require 'page'
require 'page_queue'
require 'repository'
require 'set'

# (extend to use drb to provide arbitrary function calls 
#  to the daemon?)

module SoMark
  class Crawler
    attr_reader :queue, :repository
    def initialize
      @threads = Set.new
      @debug = true
    end

    def debug(msg)
      puts msg if @debug or Thread.current[:debug]
    end

    def start(options)
      @queue = options[:queue] if options[:queue]
      @repository = options[:repository] if options[:repository]
      @freq = options[:freq] if options[:freq]
      @freq ||= 0.1
      @max_queue = options[:max_queue] if options[:max_queue]
      @max_queue ||= 20000
      @max_thread = options[:max_thread] if options[:max_thread]
      @max_thread ||= 500
      while true
        if @threads.size < @max_thread
          shift_page
        end
        sleep @freq
      end
      @repository.log "end of queue", "error"
    end
    
    def shift_page
      @threads << Thread.new do
        begin
          page = @queue.shift
          return if not page
          if @repository.saved? page
            if @queue.size == 0
              @queue.push(*page.links.collect {|l| Page.new(l)})
            end
          else
          if @repository.saving page
              download(page)
          end
          if @queue.size > @max_queue
            @queue.clean
          end
        ensure
          @threads.delete Thread.current
        end
      end      
    end
    
    def download(page)
      if page.read
        @repository.save page
        pages = (page.links.collect { |url| linked_page = Page.new(url) })
        pages = @repository.unsaved pages
        @queue.push(*pages)
      else
        # requeue failure
        @queue << page
      end
    end
  end
end