import sys
import re
import urllib,urllib2
import string
import datetime

anchor_match = re.compile(r'.*?<a.*?href="(.*?)".*?>(.*?)</a>.*?')
#ommited hrefs (apache)
ommit = ['?C=N;O=D','?C=S;O=A','?C=D;O=A','?C=M;O=A','/']
buff = ''
dump_files = True

def usage():
    print """USAGE: sPyder url_to_spider target_file [-f]\n
\turl_to_spider\tURI you want to spider
\t\t\tNOTE: if URI contains spaces you will have to put the URI string into ''
\t\t\tor replace the spaces with %20 
\ttarget_file\tfile where dir structure will be stored in
\n\tOPTIONAL parameters:
\t-f\t\tno files written in dump\n
\tEXAMPLES:
\t\tsPyder http://someserver.com/directory c:/httpdump -f
\t\tsPyder 'http://someserver.com/directory with space' c:/myshit/httpdump
\t\tsPyder http://someserver.com/directory%20with%20space c:/httpdump -f -s"""

def readIndexFile(uri = '',level = 0):
    global buff,ommit,anchor_match
    request = urllib2.Request(uri)  
    try:
        response = urllib2.urlopen(request)  
    except urllib2.HTTPError, e:
        print 'ERR: ('+str(e.code)+') Error occured. Current URI:'+uri
        buff += '\n|-'+'--'*level+uri[uri[:-1].rfind('/')+1:]+' ERR: ('+str(e.code)+')'
    except urllib2.URLError, e:
        print 'ERR: Failed to reach the URI ('+str(e.reason[0])+':'+e.reason[1]+')'
        buff += '\n|-'+'--'*level+uri[uri[:-1].rfind('/')+1:]+' ERR: ('+str(e.reason[0])+':'+e.reason[1]+')'
    else:
        for match in re.finditer(anchor_match,response.read()):
            if match.group(1) not in ommit and match.group(2) != 'Parent Directory':
                decoded_uri = urllib.unquote(match.group(1))
                if match.group(1)[-1] == '/':
                    if level > 0: buff += '\n| '+'  '*level+'|-'+decoded_uri
                    else: buff += '\n|-'+'--'*level+decoded_uri
                    readIndexFile(uri+match.group(1),level+1)
                else:
                    if(dump_files):
                        if level > 0: buff += '\n| '+'  '*level+'| '+decoded_uri
                        else: buff += '\n| '+'|--'*level+decoded_uri
#script start
if(len(sys.argv) <3):
    print 'ERR: Lack of parameters'
    usage()
else:
    try:
        f = open(sys.argv[2], 'w')
    except:
        print 'Directory does not exist or file not writable: '+sys.argv[2]
    else:
        if len(sys.argv) == 4 and sys.argv[3] == '-f': dump_files = False
        print 'Spidering URI.....'
        if sys.argv[1][-1] != '/': sys.argv[1] = sys.argv[1]+'/'
        readIndexFile(sys.argv[1])
        print 'Writing to file...'
        f.write('sPyder 1.1 by starenka <oggova@starenka.net>')
        f.write('\n@'+datetime.datetime.now().strftime("%A %d.%m.%Y %H:%M:%S")+'\n\n'+sys.argv[1])   
        f.write(buff)
        print buff+'\n'
        print 'Done ;)'