#! /usr/bin/env python from urllib2 import urlopen from BeautifulSoup import BeautifulSoup import re base_url = 'http://www.irs.gov/businesses/small/article/0,,id=104696,00.html' base = 'http://www.irs.gov' def get_state_urls(base_url): soup = BeautifulSoup(urlopen(base_url)) links = soup.fetch("table")[2].fetch("a") #stringsss = "state", "url" for link in links: return [a['href'] for a in links if a.string != "Alphabetical"] # title = link.string.replace("\n", " ") # url = link['href'] # stringsss += title, url #return title, url def get_states(base_url): soup = BeautifulSoup(urlopen(base_url)) links = soup.fetch("table")[2].fetch("a") for link in links: return [a.string.replace("\n", " ") for a in links if a.string] def get_count_info(state_url): soup = BeautifulSoup(urlopen(state_url)) # tables = soup.fetch('table')[2] county = [] # num1 = "" # num2 = "" # num3 = "" soup.prettify() # for table in tables: for tests in soup.fetch('table')[2].fetch('td'): county += tests.contents[0] + "," return county # county += table.fetch('td')[0] # num1 += table.fetch('td')[1] # num2 += table.fetch('td')[2] # num3 += table.fetch('td')[3] # table1 = county, num1, num2, num3 # return table1 # for table in tables: # return table # return [a.string.replace("\n", " ") for a in tables] # print table.find('_1') # print table.find('_2') # print table.find('_3') # return [a['td'] for a in tables] links_print = get_state_urls(base_url) state_print = get_states(base_url) for a,b in zip(links_print, state_print): url = base + a for tables in get_count_info(url): print u,'tables' # print tables[1] # print tables[2] # print tables[3] print url, b