# minimalist Web-Spider with threads
import sets, urllib, re, urlparse, thread

# Used for threading
def process_queue(id):
  global queue, visited
  while True:
    while not queue: pass
    url = queue.pop(0)

    print 'Thread %d Doing %s ' % (id, url)
    try:
      sock = urllib.urlopen(url)
      page = sock.read()
      sock.close()
    except IOError:
      continue

    for match in re.findall(r'href=\"(.+?)\"', page):
      candidate = urlparse.urljoin(url, match)
      if candidate in visited: continue
      queue.append(candidate)
      visited.add(candidate)

queue = ['http://www.oregonstate.edu']
visited = sets.Set()
for i in range(20):
  thread.start_new_thread(process_queue, (i,))
while True: pass