# $Idaemons: /home/cvs/pkgtools/lib/portsdb.rb,v 1.65 2002/04/28 19:56:59 knu Exp $

require 'singleton'
require 'tempfile'
require 'fnmatch' unless File.respond_to?(:fnmatch?)
require 'pkgmisc'

class PortsDB
  include Singleton
  include Enumerable

  DB_VERSION = [:FreeBSD, 3]

  LANGUAGE_SPECIFIC_CATEGORIES = {
    "chinese"		=> "zh-",
    "french"		=> "fr-",
    "german"		=> "de-",
    "hebrew"		=> "iw-",
    "japanese"		=> "ja-",
    "korean"		=> "ko-",
    "russian"		=> "ru-",
    "ukrainian"		=> "uk-",
    "vietnamese"	=> "vi-",
  }

  MY_PORT = 'sysutils/portupgrade'

  attr_accessor :ignore_categories, :extra_categories

  class IndexFileError < StandardError
#    def message
#      "index file error"
#    end
  end

  class DBError < StandardError
#    def message
#      "database file error"
#    end
  end

  def setup(alt_db_dir = nil, alt_ports_dir = nil, alt_db_driver = nil)
    set_ports_dir(alt_ports_dir)
    set_db_dir(alt_db_dir)
    set_db_driver(alt_db_driver)

    @categories = nil
    @virtual_categories = nil
    @ignore_categories = []
    @extra_categories = []
    @origins = nil
    @pkgnames = nil
    @origins_by_categories = {}
    @ports = {}
    @localbase = nil
    @x11base = nil

    self
  end

  def ports_dir()
    unless @ports_dir
      set_ports_dir(nil)	# initialize with the default value
    end

    @ports_dir
  end

  def ports_dir=(new_ports_dir)
    @abs_ports_dir = @index_file = nil

    @ports_dir = new_ports_dir || ENV['PORTSDIR'] || '/usr/ports'
  end
  alias set_ports_dir ports_dir=

  def abs_ports_dir()
    unless @abs_ports_dir
      dir = ports_dir

      Dir.chdir(dir) {
	@abs_ports_dir = Dir.pwd
      } rescue raise DBError, "Can't chdir to '#{dir}'"
    end

    @abs_ports_dir
  end

  def db_driver()
    unless @db_driver
      set_db_driver(nil)	# initialize with the default value
    end

    @db_driver
  end

  def db_driver=(new_db_driver)
    begin
      case new_db_driver || ENV['PORTS_DBDRIVER'] || 'bdb1_btree'
      when 'bdb_btree'
	@db_driver = :bdb_btree
      when 'bdb_hash', 'bdb'
	@db_driver = :bdb_hash
      when 'bdb1_btree', 'btree'
	@db_driver = :bdb1_btree
      when 'bdb1_hash', 'hash', 'bdb1'
	@db_driver = :bdb1_hash
      else
	@db_driver = :dbm_hash
      end

      case @db_driver
      when :bdb_btree
	next_driver = 'bdb1_btree'
	require 'bdb'
	@db_params = ["set_pagesize" => 1024, "set_cachesize" => [0, 32 * 1024, 0]]
      when :bdb_hash
	next_driver = 'bdb1_hash'
	require 'bdb'
	@db_params = ["set_pagesize" => 1024, "set_cachesize" => [0, 32 * 1024, 0]]
      when :bdb1_btree
	next_driver = 'dbm'
	require 'bdb1'
	@db_params = ["set_pagesize" => 1024, "set_cachesize" => 32 * 1024]
      when :bdb1_hash
	next_driver = 'dbm'
	require 'bdb1'
	@db_params = ["set_pagesize" => 1024, "set_cachesize" => 32 * 1024]
      else
	next_driver = nil
	require 'dbm'
      end
    rescue LoadError
      if next_driver.nil?
	raise DBError, "No driver is available!"
      end

      new_db_driver = next_driver
      retry
    end

    @db_driver
  end
  alias set_db_driver db_driver=

  def index_file()
    unless @index_file
      @index_file = ENV['PORTS_INDEX'] || portdir('INDEX')
    end

    @index_file
  end

  def db_dir()
    unless @db_dir
      set_db_dir(nil)	# initialize with the default value
    end

    @db_dir
  end

  def db_dir=(new_db_dir)
    @db_dir = new_db_dir || ENV['PORTS_DBDIR'] || ports_dir

    @db_file = File.join(@db_dir, 'INDEX.db')
    @db_filebase = @db_file.sub(/\.db$/, '')

    close_db

    @db_dir
  end
  alias set_db_dir db_dir=

  def db_dir_list()
    [
      db_dir,
      ports_dir,
      PkgDB.instance.db_dir,
      ENV['TMPDIR'],
      '/var/tmp',
      '/tmp'
    ].compact
  end

  def localbase
    @localbase ||= `cd #{portdir(MY_PORT)} && make -V LOCALBASE`.strip
  end

  def x11base
    @x11base ||= `cd #{portdir(MY_PORT)} && make -V X11BASE`.strip
  end

  def join(category, port)
    File.join(category, port)
  end

  def split(origin)
    if %r"^([^./A-Z][^/]*)/([^./][^/]*)$" =~ path
      return $1, $2
    end

    nil
  end

  def strip(path, existing_only = false)
    path = path.tr_s('/', '/')

    %r"^(?:(/.+)/)?([^./][^/]*/[^./][^/]*)/?$" =~ path or return nil

    dir = $1
    port = $2

    if dir && dir != ports_dir && dir != abs_ports_dir
      return nil
    end

    if existing_only && !exist?(port)
      return nil
    end

    port
  end

  def portdir(port)
    File.join(ports_dir, port)
  end

  def subdirs(dir)
    %x"fgrep -v bsd.port.subdir.mk #{dir}/Makefile |
       make -f - -V SUBDIR 2> /dev/null".split.select { |i|
      File.directory?(File.join(dir, i))
    }.sort
  end

  def categories
    open_db if @categories.nil?

    @categories
  end

  def real_categories!
    subdirs(ports_dir)
  end

  def categories!
    customize_categories(real_categories!)
  end

  def customize_categories(cats)
    ((cats | @extra_categories) - @ignore_categories).sort
  end

  def category?(category)
    @categories.qinclude?(category)
  end

  def virtual_categories
    open_db if @virtual_categories.nil?

    @virtual_categories
  end

  def virtual_category?(category)
    @virtual_categories.qinclude?(category)
  end

  def ignore_category?(category)
    @ignore_categories.qinclude?(category)
  end

  def update
    STDERR.print "Updating the ports index ... "
    STDERR.flush

    t = Tempfile.new('INDEX')
    t.close
    tmp = t.path

    if File.exist?(index_file)
      if !File.writable?(index_file)
	STDERR.puts "index file #{index_file} not writable!"
	raise IndexFileError, "index generation error"
      end
    else
      dir = File.dirname(index_file)

      if !File.writable?(dir)
	STDERR.puts"index file directory #{dir} not writable!"
	raise IndexFileError, "index generation error"
      end
    end

    if true
      system %`
	#{PkgDB::CMD[:make_describe_pass1]} |
	#{PkgDB::CMD[:make_describe_pass2]} |
	(cd #{abs_ports_dir} && perl Tools/make_index) |
	sed -e 's./.\001.g' |
	sort -t '|' +1 -2 |
	sed -e 's.\001./.g' > #{tmp}
      `
    else
      system %`
	cd #{abs_ports_dir} && (make describe ECHO_MSG='echo > /dev/null' 2> /dev/null |
	perl Tools/make_index) |
	sed -e 's./.\001.g' |
	sort -t '|' +1 -2 |
	sed -e 's.\001./.g' > #{tmp}
      `
    end

    if File.zero?(tmp)
      STDERR.puts 'failed to generate INDEX!'
      raise IndexFileError, "index generation error"
    end

    begin
      File.chmod(0644, tmp)
    rescue => e
      STDERR.puts e.message
      raise IndexFileError, "index chmod error"
    end

    if not system('/bin/mv', '-f', tmp, index_file)
      STDERR.puts 'failed to overwrite #{index_file}!"'
      raise IndexFileError, "index overwrite error"
    end

    STDERR.puts "done"

    @categories = nil
    @virtual_categories = nil
    @origins = nil
    @pkgnames = nil
    @origins_by_categories = {}
    @ports = {}

    close_db
  end

  def open_db_for_read!
    close_db

    case db_driver
    when :bdb_btree
      @db = BDB::Btree.open @db_file, nil, 'r', 0, *@db_params
    when :bdb_hash
      @db = BDB::Hash.open @db_file, nil, 'r', 0, *@db_params
    when :bdb1_btree
      @db = BDB1::Btree.open @db_file, 'r', 0, *@db_params
    when :bdb1_hash
      @db = BDB1::Hash.open @db_file, 'r', 0, *@db_params
    else
      @db = DBM.open(@db_filebase)
    end
  end

  def open_db_for_update!
    close_db

    case db_driver
    when :bdb_btree
      @db = BDB::Btree.open @db_file, nil, 'r+', 0664, *@db_params
    when :bdb_hash
      @db = BDB::Hash.open @db_file, nil, 'r+', 0664, *@db_params
    when :bdb1_btree
      @db = BDB1::Btree.open @db_file, 'r+', 0664, *@db_params
    when :bdb1_hash
      @db = BDB1::Hash.open @db_file, 'r+', 0664, *@db_params
    else
      @db = DBM.open(@db_filebase)
    end
  end

  def open_db_for_rebuild!
    close_db

    if File.exist?(rbo = @db_filebase + '.rbo')
      STDERR.print "[INDEX.rbo is no longer needed.  Removing]"
      File.unlink(rbo)
    end

    case db_driver
    when :bdb_btree
      @db = BDB::Btree.open @db_file, nil, 'w+', 0664, *@db_params
    when :bdb_hash
      @db = BDB::Hash.open @db_file, nil, 'w+', 0664, *@db_params
    when :bdb1_btree
      @db = BDB1::Btree.open @db_file, 'w+', 0664, *@db_params
    when :bdb1_hash
      @db = BDB1::Hash.open @db_file, 'w+', 0664, *@db_params
    else
      File.unlink(@db_file) if File.exist?(@db_file)

      @db = DBM.open(@db_filebase, 0664)
    end
  end

  def open_db
    @db and return @db

    update_db

    retried = false

    begin
      open_db_for_read!

      check_db_version or raise TypeError, 'database version mismatch/bump detected'

      s = @db[':categories']
      s.is_a?(String) or raise TypeError, "missing key: categories"
      @categories = s.split

      s = @db[':virtual_categories']
      s.is_a?(String) or raise TypeError, "missing key: virtual_categories"
      @virtual_categories = s.split

      s = @db[':origins']
      s.is_a?(String) or raise TypeError, "missing key: origins"
      @origins = s.split

      s = @db[':pkgnames']
      s.is_a?(String) or raise TypeError, "missing key: pkgnames"
      @pkgnames = s.split.map { |n| PkgInfo.new(n) }

      s = @db[':virtual_categories']
      s.is_a?(String) or raise TypeError, "missing key: virtual_categories"
      @virtual_categories = s.split

      @origins_by_categories = {}
      (@categories + @virtual_categories).each do |c|
	s = @db['?' + c] and @origins_by_categories[c] = s.split
      end
    rescue => e
      if retried
	raise DBError, "#{e.message}: Cannot read the portsdb!"
      end

      STDERR.print "[#{e.message}] "
      update_db(true)

      retried = true
      retry
    end

    @ports = {}

    @db
  rescue => e
    STDERR.puts e.message
    raise DBError, 'database file error'
  end

  def close_db
    if @db
      @db.close
      @db = nil
    end
  end

  def date_index
    File.mtime(index_file) rescue nil
  end

  def date_db
    File.mtime(@db_file) rescue nil
  end

  def up_to_date?
    d1 = date_db() and d2 = date_index() and d1 >= d2
  end

  def check_db_version
    db_version = Marshal.load(@db[':db_version'])

    db_version[0] == DB_VERSION[0] && db_version[1] == DB_VERSION[1]
  rescue => e
    return false
  end

  def select_db_dir(force = false)
    return db_dir if File.writable?(db_dir)

    db_dir_list.each do |dir|
      set_db_dir(dir)

      !force && up_to_date? and return dir

      File.writable?(dir) and return dir
    end

    nil
  end

  def update_db(force = false)
    update if not File.exist?(index_file)

    !force && up_to_date? and return false

    close_db

    select_db_dir(force) or raise "No directory available for portsdb!"

    prev_sync = STDERR.sync
    STDERR.sync = true

    STDERR.printf "[Updating the portsdb <format:%s> in %s ... ", db_driver, db_dir

    nports = `wc -l #{index_file}`.to_i

    STDERR.printf "- %d port entries found ", nports

    i = -1

    @origins = []
    @pkgnames = []

    begin
      open_db_for_rebuild!

      File.open(index_file) do |f|
	f.each_with_index do |line, i|
	  lineno = i + 1

	  if lineno % 100 == 0
	    if lineno % 1000 == 0
	      STDERR.print lineno
	    else
	      STDERR.putc(?.)
	    end
	  end

	  begin
	    port_info = PortInfo.new(line)

	    next if ignore_category?(port_info.category)

	    origin = port_info.origin
	    pkgname = port_info.pkgname

	    port_info.categories.each do |category|
	      if @origins_by_categories.key?(category)
		@origins_by_categories[category] << origin
	      else
		@origins_by_categories[category] = [origin]
	      end
	    end
	    
	    @ignore_categories.each do |category|
	      @origins_by_categories.delete(category)
	    end

	    @origins << origin
	    @pkgnames << pkgname

	    @db[origin] = port_info
	    @db[pkgname.to_s] = origin
	  rescue => e
	    STDERR.puts index_file + ":#{lineno}:#{e.message}"
	  end
	end
      end

      STDERR.print ' '

      real_categories = real_categories! | @extra_categories
      all_categories = @origins_by_categories.keys

      @categories = (real_categories - @ignore_categories).sort
      @virtual_categories = (all_categories - real_categories).sort

      @db[':categories'] = @categories.join(' ')
      STDERR.putc(?.)
      @db[':virtual_categories'] = @virtual_categories.join(' ')
      STDERR.putc(?.)
      @db[':origins'] = @origins.join(' ')
      STDERR.putc(?.)
      @db[':pkgnames'] = @pkgnames.map { |n| n.to_s }.join(' ')
      STDERR.putc(?.)
      all_categories.each do |c|
	@db['?' + c] = @origins_by_categories[c].join(' ')
      end
      STDERR.putc(?.)
      @db[':db_version'] = Marshal.dump(DB_VERSION)
    rescue => e
      File.unlink(@db_file) if File.exist?(@db_file)
      raise DBError, "#{e.message}: Cannot update the portsdb! (#{@db_file})]"
    ensure
      close_db
    end

    STDERR.puts " done]"
    STDERR.sync = prev_sync

    true
  end

  def port(key)
    key.is_a?(PortInfo) and return key

    @ports.key?(key) and return @ports[key]

    open_db

    if key.include?('/')
      val = @db[key]
    elsif val = @db[key]
      return port(val)
    end

    @ports[key] = if val then PortInfo.new(val) else nil end
  end
  alias [] port

  def ports(keys)
    keys.map { port(key) }
  end

  alias indices ports

  def origin(key)
    if p = port(key)
      p.origin
    else
      nil
    end
  end

  def origins(category = nil)
    open_db

    if category
      @origins_by_categories[category]
    else
      @origins
    end
  end

  def origins!(category = nil)
    if category
      # only lists the ports which primary category is the given category
      subdirs(portdir(category)).map { |i|
	File.join(category, i)
      }
    else
      list = []

      categories!.each do |i|
	list.concat(origins!(i))
      end

      list
    end
  end

  def each(category = nil)
    ports = origins(category) or return nil

    ports.each { |key|
      yield(@db[key])
    }
  end

  def each_category
    categories.each { |key|
      yield(key)
    }
  end

  def each_origin(category = nil)
    ports = origins(category) or return nil

    ports.each { |key|
      yield(key)
    }
  end

  def each_origin!(category = nil, &block)
    if category
      # only lists the ports which primary category is the given category
      subdirs(portdir(category)).each do |i|
	block.call(File.join(category, i))
      end
    else
      categories!.each do |i|
	each_origin!(i, &block)
      end
    end
  end

  def each_pkgname
    open_db

    @pkgnames.each { |key|
      yield(key)
    }
  end

  def glob(pattern = '*')
    list = []
    pkg = nil

    open_db

    case pattern
    when Regexp
      is_port = pattern.source.include?('/')
    else
      if /^[<>]/ =~ pattern
	raise "Invalid glob pattern: #{pattern}"
      end

      is_port = pattern.include?('/')

      # shortcut
      if portinfo = port(pattern)
	if block_given?
	  yield(portinfo)
	  return nil
	else
	  return [portinfo]
	end
      end
    end

    if is_port
      @origins.each do |origin|
	case pattern
	when Regexp
	  next if pattern !~ origin
	else
	  next if not File.fnmatch?(pattern, origin, File::FNM_PATHNAME)
	end

	if portinfo = port(origin)
	  if block_given?
	    yield(portinfo)
	  else
	    list.push(portinfo)
	  end
	end
      end
    else
      @pkgnames.each do |pkgname|
	next if not pkgname.match?(pattern)

	if portinfo = port(pkgname.to_s)
	  if block_given?
	    yield(portinfo)
	  else
	    list.push(portinfo)
	  end
	end
      end
    end

    if block_given?
      nil
    else
      list
    end
  rescue => e
    STDERR.puts e.message

    if block_given?
      return nil
    else
      return []
    end
  end

  def exist?(port, quick = false)
    return if %r"^[^/]+/[^/]+$" !~ port

    dir = portdir(port)

    return false if not File.file?(File.join(dir, 'Makefile'))

    return true if quick

    pkgname = `cd #{dir} && make -V PKGNAME 2>/dev/null`.chomp

    if pkgname.empty?
      false
    else
      pkgname
    end
  end

  def all_depends_list!(origin, before_args = nil, after_args = nil)
    `cd #{$portsdb.portdir(origin)} && #{before_args || ''} make #{after_args || ''} all-depends-list`.map { |line|
      strip(line.chomp, true)
    }.compact
  end

  def all_depends_list(origin, before_args = nil, after_args = nil)
    if !before_args && !after_args && i = port(origin)
      i.all_depends.map { |n| origin(n) }
    else
      all_depends_list!(origin, before_args, after_args)
    end
  end

  def masters(port)
    dir = portdir(port)

    ports = []

    `cd #{dir} ; make -dd -n 2>&1`.each do |line|
      if /^Searching for .*\.\.\.Caching .* for (\S+)/ =~ line.chomp
	path = File.expand_path($1)

	if (path.sub!(%r"^#{Regexp.quote(ports_dir)}/", '') ||
	    path.sub!(%r"^#{Regexp.quote(abs_ports_dir)}/", '')) &&
	    %r"^([^/]+/[^/]+)" =~ path
	  x = $1

	  ports << x if exist?(x) && !ports.include?(x)
	end
      end
    end

    ports.delete(port)

    ports
  end

  def latest_link(port)
    dir = portdir(port)

    flag, name = `cd #{dir} && make -V NO_LATEST_LINK -V LATEST_LINK`.to_a

    if flag.chomp.empty?
      name.chomp
    else
      nil
    end
  end

  def sort(ports)
    tsort = TSort.new

    ports.each do |p|
      portinfo = port(p)

      portinfo or next

      o = portinfo.origin

      deps = all_depends_list(o)	# XXX

      tsort.add(o, *deps)
    end

    tsort.tsort! & ports
  end

  def sort!(ports)
    ports.replace(sort(ports))
  end

  def recurse(portinfo, recurse_down = false, recurse_up = false)
    if not portinfo.is_a?(PortInfo)
      portinfo = port(portinfo)
    end

    list = []

    portinfo or return list

    if recurse_up
      portinfo.all_depends.map do |name|
	i = port(name)

	list << i if i
      end
    end

    list << portinfo

    if recurse_down
      # slow!
      pkgname = portinfo.pkgname.fullname

      glob do |i|
	list << i if i.all_depends.include?(pkgname)
      end
    end

    list
  end
end
