#!/usr/bin/env ruby
#
# $Id: webdata,v 1.2 2003/07/13 12:09:42 danbri Exp $
# A sample scutter implementation, using the framework 
# defined in RDF4R/scutter.rb
#
# by Dan Brickley <danbri@w3.org>
# $Id: webdata,v 1.2 2003/07/13 12:09:42 danbri Exp $
#
# TODO: this is more a demo app than a test. write some tests and 
#       turn the rest into a general RDF utility eg /usr/bin/webdata
#  ...move to ../examples/ ?
#

help_long=<<EOT;
webdata utility
This utility provides basic RDF load, parse, store and harvest facilities.

When run in --scutter mode, rdfs:seeAlso links between RDF Web pages will 
be traversed.
Example:
webdata  --scutter=http://swordfish.rdfweb.org/discovery/2001/08/codepict/scutterplan.rdf
EOT

help_setup=<<EOT;

To test, we need to create a fresh database and populate it. 
The db creation script will complain if it exists already. Note that 
initdb-pg.sql will DESTROY any existing content in the named database. 
USE WITH CARE!.
  
Here is the setup we use for smush tests with the FOAF smush tests:

  createdb --quiet foafaz; psql -q foafaz < ../../../db/initdb-pg.sql
  ./webdata -n DBI:Pg:foafaz -s http://rdfweb.org/2003/02/a-z/az-scutterplan.rdf

If we know the database already exists, and we want to DESTROY it for 
testing purposes, we use:

  ./webdata --destroydb -n DBI:Pg:foafaz -s http://rdfweb.org/2003/02/a-z/az-scutterplan.rdf

  TODO: perhaps we should only allow the commandline --destroydb to work with 
  databases whose name contains the string 'test', or similar? 

EOT

# How it works:
# Each 'lambda' is basically a Proc.new creating a code fragment that 
# gets called at various lifecycle points by the Web harvester, passing
# the crawler and current doc. This may or may not be the best way to 
# do things, but seems pretty flexible. Need to think about how this 
# might interact with threading, and find out if Ruby is ever going to get
# genuine threads (useful when tcp/ip stuff blocks, eg dns lookups?).
#
# So we basically create a new 'Scutter', register some code, and set it
# running. 
#
# basic idea in 8 lines - see tiny.rb; also library docs)
#
# What it does:
# BE WARNED, this code has SIDE EFFECTS.
# - it will try to write to a PostgreSQL triple store
# - it will make a mess of your stdout
# - it will make network connections to random HTTP sites it finds
# - it (on windows at least) may chew up memory resources
#
# On the other hand, it is kind of cute.
# 
# Why it does it:
#
# To show what a basic RDF harvester might do, eg. to acquire data from the 
# Web, and to show where we might plug in smarter traversal heuristics. Current
# library is hardcoded to simply track all the rdfs:seeAlso crossrefs it finds.
#
# This might be of interest if you want to load some data into a local
# store/cache for more efficient query. See the Cwm pastich at 
# http://www.w3.org/2001/12/rubyrdf/pack/tests/scutter/sqlgraph.rb for 
# possible example client code. A FOAFNaut (see http://www.foafnaut.org/)
# aggregator using this approach is planned (once API to db fixed up).

$LOAD_PATH.unshift '../../lib/' # dev't version, use local copy of library.

require 'basicrdf'
require 'squish'
require 'RDF4R/scutter'
require 'getoptlong'



## Given a start URI and some options, set up a crawler and go...
#
def go(uri, o={})
  foaf='http://xmlns.com/foaf/0.1/' 
  contact="http://www.w3.org/2000/10/swap/pim/contact#" 
  air='http://www.megginson.com/exp/ns/airports#'
  #todo: geo?

  ayf = RDF4R::SimpleScutter.new uri
  pagecount=0

  # a code block to output basic info about each RDF page encountered
  # 
  page_summary = lambda do |crawler,page|  
    puts "RDFDOC: count='#{pagecount}': uri:#{crawler.uri} gave RDF graph #{page} \
	with #{page.size} triples\n" 
    pagecount=pagecount+1
  end

  # a code block to see if some page provides nearestAirport information:  
  #
  #  <contact:nearestAirport><wn:Airport air:icao="EGGD" air:iata="BRS"/>...
  #
  airports = lambda do |crawler,page| 

    #page.reg_xmlns 'http://www.megginson.com/exp/ns/airports#', 'air'
   
    rs = page.ask Statement.new(nil, air+"iata", nil)
    rs.objects.each do |a|
      a.graph=page
      puts "AIRPORT: #{a} got airport code in #{crawler.uri})"
    end
  end

  # stats to be output at start of each loop      
  #
  loopstats = lambda do |s|
    puts "INIT: s.left.size=#{s.left.size} s.seen.size=#{s.seen.size} current: #{s.uri}"
  end

  error_logger = lambda {|e| puts "ERROR: #{e}" }


  #trying to find src of memory bloat: (on windows anyway)
  #objstats=nil
  #ayf.pagehandlers.push Proc.new {|c,page| 
  #  objstats=Hash.new(0) # empty and recount each loop
  #  ObjectSpace.each_object{|x| 
  #    objstats[x.class.name]=objstats[x.class.name]+1
  #  }
  #  puts "ObjectStats: #{objstats.inspect}\n"
  #}

#  puts "OPTIONS are: #{o.inspect}"
  #o['--dbi-user']='foooo'
#  puts "USER was: #{o['--dbi-user']}"
  n='DBI:Pg:scutter1'
  u='danbri'
  p=''
  n = o['--dbi-driver'] if o['--dbi-driver']
  u = o['--dbi-user'] if o['--dbi-user'] 
  p = o['--dbi-pass'] if o['--dbi-pass']
#  n = (o['--dbi-driver'] or 'DBI:Pg:scutter1')
#  u = (o['--dbi-user'] or 'danbri')
#  p = (o['--dbi-pass'] or '')

  puts "\n\nDB config: -n=#{n} -u=#{u} -p=#{p} \n\nopts:#{o.inspect}\n\n"

  sink = SimpleSQLGraph.new('dbi_driver'=>n,'dbi_user'=>u, 'dbi_pass'=>p)
  sink.verbosity=5

  if o['--destroydb']
    puts "DESTROYING DATABASE:"+sink.destroydb
  end

  datasink = lambda do |crawler,page|
    puts "Storing data from #{crawler.uri}"
    puts "SQL log: "+sink.store_graph(page, crawler.uri)
    puts "Smushing..."
    puts sink.defrag
    sink.addAllSuperProperties #todo
    puts "SQLdb: status: #{sink.status}"

  end


  # register some handlers:
  ayf.pagehandlers.push page_summary, airports, datasink
  ayf.inithandlers.push loopstats
  ayf.errorhandlers.push error_logger 
  ayf.run  # set crawler running!
end 

################################################################# 


c = GetoptLong.new (
                   ['--check', '-c', GetoptLong::REQUIRED_ARGUMENT],
                   ['--destroydb','-x', GetoptLong::NO_ARGUMENT],
                   ['--test','-t', GetoptLong::NO_ARGUMENT],
                   ['--collect','-f', GetoptLong::OPTIONAL_ARGUMENT],
                   ['--scutter','-s', GetoptLong::REQUIRED_ARGUMENT],
                   ['--dbi-driver','-n', GetoptLong::REQUIRED_ARGUMENT],
                   ['--dbi-user','-u', GetoptLong::REQUIRED_ARGUMENT],
                   ['--dbi-pass','-p', GetoptLong::REQUIRED_ARGUMENT]
                   )


#start_uri = 'http://rdfweb.org/people/danbri/rdfweb/danbri-foaf.rdf' 
o={}
c.each_option do |name,arg|
  arg.chomp!
  puts "#{name} -> #{arg}"
  o[name]=arg if arg
end
puts "INITIAL options: #{o.inspect}"
start_uri = o['--scutter']
go(start_uri,o)


#  ./ayftest.rb --scutter=http://rdfweb.org/people/danbri/rdfweb/danbri-foaf.rdf -n DBI:Pg:scutter1 --dbi-user=danbri


# select distinct o, count(o) from simple where o like 'http://xmlns.com/wordnet/1.6/%' group by o ORDER BY count(o) desc;