cat citeseerdownloaderv2.py 
#TODO: 
#       - Te puede aparecer un    <header status="deleted">  
#       - CLASSPATH="/home/gus/workspace/CiteseerDownload/src/OAI-PMH.jar:/home/gus/xmlbeans-2.2.0/lib/xmlpublic.jar:/home/gus/xmlbeans-2.2.0/lib/xbean.jar:/home/gus/xmlbeans-2.2.0/lib/xbean_xpath.jar:/home/gus/xmlbeans-2.2.0/lib/jsr173_1.0_api.jar:/home/gus/workspace/CiteseerDownload/src/oai_dc.jar"
#       - Capas que lo mejor seria generar un YAML
import httplib, os
import org.openarchives.oai.x20 
import java.io.File
import org.apache.xmlbeans

def get_listing(conn):
        conn.request("GET", "/oai2?verb=ListIdentifiers&metadataPrefix=oai_dc")
        response = conn.getresponse()
        if (response.reason == "OK" and response.status == 200):
                listidsdoc = org.openarchives.oai.x20.OAIPMHDocument.Factory.parse(response.read())
                listids = listidsdoc.getOAIPMH().getListIdentifiers().getHeaderArray()
                return listids
        else:
                print "Problema obtenindo la lista de archivos"

def get_paper(id, i, out):
        conn = httplib.HTTPConnection("export.arxiv.org")
        url = "/oai2?verb=GetRecord&identifier=%s&metadataPrefix=oai_dc" % id
        conn.request("GET", url)
        response = conn.getresponse()
        xmlopts = org.apache.xmlbeans.XmlOptions()
        xmlopts.setSaveUseOpenFrag() 
        xmlopts.setUseDefaultNamespace() 
        rec = org.openarchives.oai.x20.OAIPMHDocument.Factory.parse(response.read(),xmlopts)
        meta = rec.getOAIPMH().getGetRecord().getRecord().getMetadata()
        dc = org.openarchives.oai.x20.oaiDc.DcDocument.Factory.parse(meta.toString(),xmlopts).getDc()
        title = dc.getTitleArray(0).toString().split("<xml-fragment xmlns:oai_dc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">")[1].split("</xml-fragment>")[0]
        dsc = dc.getDescriptionArray(0).toString().split("<xml-fragment xmlns:oai_dc=\"http://www.openarchives.org/OAI/2.0/oai_dc/\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\">")[1].split("</xml-fragment>")[0]
        name = id.split(":")
        name = name[2]
        urlpdf = "http://arxiv.org/pdf/%s" % name
        os.system("curl %s > /home/gus/papersV2/download/%s.pdf" % (urlpdf, name))
        out.write("<File number=\"%d\" title=\"%s\" name=\"%s\" abstract=\"%s\" > </File> \n" % (i, title, name, dsc))
        out.flush()
        conn.close()


def get_papers(out):
        """ Main Loop """
        out.write('<?xml version="1.0" encoding="ISO-8859-1"?> \n')    
        out.write('<root> \n')
        conn = httplib.HTTPConnection("export.arxiv.org")
        listids = get_listing(conn)
        i=0
        for id in listids[0:len(listids)/6]: #hay 10k en este repo vamo a encarar la tercera parte
                i=i+1
                print "%d/%d" % (i,len(listids)/6)
                get_paper(id.getIdentifier(), i, out)
        out.write('</root>')
        out.close()

out = open ('/home/gus/papersV2/download/summary.out', 'w') 
get_papers(out)