#!/usr/bin/env ruby

#--
# Copyright (c) 2006 Jens Kraemer
# 
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
# 
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
# 
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#++
#

RDIGVERSION = '0.3.4'


require 'thread'
require 'thwait'
require 'singleton'
require 'monitor'
require 'ostruct'
require 'uri'
require 'cgi'
require 'set'
require 'net/http'
require 'getoptlong'
require 'tempfile'
require 'open-uri'

begin
  require 'ferret'
rescue LoadError
  require 'rubygems'
  require 'ferret'
end

require 'htmlentities/htmlentities'


$KCODE = 'u'
require 'jcode'

# See README for basic usage information
module RDig

  class << self

    # the filter chains are for limiting the set of indexed documents.
    # there are two chain types - one for http, and one for file system
    # crawling.
    # a document has to survive all filters in the chain to get indexed.
    def filter_chain
      @filter_chain ||= {
        # filter chain for http crawling
        :http => [
          :scheme_filter_http,
          :fix_relative_uri,
          :normalize_uri,
          { :hostname_filter => :include_hosts },
          { RDig::UrlFilters::UrlInclusionFilter => :include_documents },
          { RDig::UrlFilters::UrlExclusionFilter => :exclude_documents },
          RDig::UrlFilters::VisitedUrlFilter 
        ],
        # filter chain for file system crawling
        :file => [
          :scheme_filter_file,
          { RDig::UrlFilters::PathInclusionFilter => :include_documents },
          { RDig::UrlFilters::PathExclusionFilter => :exclude_documents }
        ]
      }
         
    end

    def application
      @application ||= Application.new
    end

    def searcher
      @searcher ||= Search::Searcher.new(config.index)
    end

    # RDig configuration
    #
    # may be used with a block:
    #   RDig.configuration do |config| ...
    #
    # see doc/examples/config.rb for a commented example configuration
    def configuration
      if block_given?
        yield configuration
      else
        @config ||= OpenStruct.new(
          :crawler           => OpenStruct.new(
            :start_urls        => [ "http://localhost:3000/" ],
            :include_hosts     => [ "localhost" ],
            :include_documents => nil,
            :exclude_documents => nil,
            :index_document    => nil,
            :num_threads       => 2,
            :max_redirects     => 5,
            :wait_before_leave => 10
          ),
          :content_extraction  => OpenStruct.new(
            # settings for html content extraction (hpricot)
            :hpricot      => OpenStruct.new(
              # css selector for the element containing the page title
              :title_tag_selector => 'title', 
              # might also be a proc returning either an element or a string:
              # :title_tag_selector => lambda { |hpricot_doc| ... }
              :content_tag_selector => 'body'
              # might also be a proc returning either an element or a string:
              # :content_tag_selector => lambda { |hpricot_doc| ... }
            )
            #,
            # # settings for html content extraction (RubyfulSoup)
            # :rubyful_soup => OpenStruct.new(
            #  # select the html element that contains the content to index
            #  # by default, we index all inside the body tag:
            #  :content_tag_selector => lambda { |tagsoup|
            #    tagsoup.html.body
            #  },
            #  # select the html element containing the title 
            #  :title_tag_selector         => lambda { |tagsoup|
            #    tagsoup.html.head.title
            #  }
            # )
          ),
          :index                 => OpenStruct.new( 
            :path                => "index/", 
            :create              => true,
            :handle_parse_errors => true,
            :analyzer            => Ferret::Analysis::StandardAnalyzer.new,
            :occur_default       => :must,
            :default_field       => '*'
          )
        )
      end
    end
    alias config configuration
    
  end

  class Application

    OPTIONS = [
      ['--config',   '-c', GetoptLong::REQUIRED_ARGUMENT,
        "Read aplication configuration from CONFIG."],
      ['--help',     '-h', GetoptLong::NO_ARGUMENT,
        "Display this help message."],
      ['--query',   '-q', GetoptLong::REQUIRED_ARGUMENT,
        "Execute QUERY."],
      ['--version',  '-v', GetoptLong::NO_ARGUMENT,
       	"Display the program version."],
    ]

    # Application options from the command line
    def options
      @options ||= OpenStruct.new
    end
    
    # Display the program usage line.
    def usage
      puts "rdig -c configfile {options}"
    end
    
    # Display the rake command line help.
    def help
      usage
      puts
      puts "Options are ..."
      puts
      OPTIONS.sort.each do |long, short, mode, desc|
        if mode == GetoptLong::REQUIRED_ARGUMENT
          if desc =~ /\b([A-Z]{2,})\b/
            long = long + "=#{$1}"
          end
        end
        printf "  %-20s (%s)\n", long, short
        printf "      %s\n", desc
      end
    end

    # Return a list of the command line options supported by the
    # program.
    def command_line_options
      OPTIONS.collect { |lst| lst[0..-2] }
    end

    # Do the option defined by +opt+ and +value+.
    def do_option(opt, value)
      case opt
      when '--help'
        help
        exit
      when '--config'
        options.config_file = value
      when '--query'
        options.query = value
      when '--version'
        puts "rdig, version #{RDIGVERSION}"
        exit
      else
        fail "Unknown option: #{opt}"
      end 
    end

    # Read and handle the command line options.
    def handle_options
      opts = GetoptLong.new(*command_line_options)
      opts.each { |opt, value| do_option(opt, value) }
    end

    # Load the configuration
    def load_configfile
      load File.expand_path(options.config_file)
    end

    # Run the +rdig+ application.
    def run
      puts "RDig version #{RDIGVERSION}"
      handle_options
      begin
        load_configfile
      rescue
        puts $!.backtrace
        fail "No Configfile found!\n#{$!}"
        
      end    

      puts "using Ferret #{Ferret::VERSION}"

      if options.query
        # query the index
        puts "executing query >#{options.query}<"
        results = RDig.searcher.search(options.query)
        puts "total results: #{results[:hitcount]}"
        results[:list].each { |result|
          puts <<-EOF
#{result[:url]}
  #{result[:title]}
  #{result[:extract]}

          EOF
        }
      else
        # rebuild index
        @crawler = Crawler.new
        @crawler.run
      end
    end
  end
end

require 'rdig/content_extractors'
require 'rdig/url_filters'
require 'rdig/search'
require 'rdig/index'
require 'rdig/file'
require 'rdig/documents'
require 'rdig/crawler'