package AsciiDatabase;
import ModSQL.*;
import java.sql.*;
import java.io.*;
import java.util.Vector;
import java.util.Random;
import java.util.zip.*;
import java.lang.System;

/* AsciiDatabase/AsciiTable.java
 *
 * Copyright (c) 2003 Chris Studholme <chris.studholme@utoronto.ca>
 *
 * May be copied or modified under the terms of the GNU General Public
 * License.  See COPYING for more information.
 */

/**
 * <p>Properties of an individual column of the table.
 *
 * @author chris.studholme@utoronto.ca
 */
final class ColumnProperties implements Serializable {
  /** @serial name of column */
  public String name;
  /** @serial SQL type of column */
  public int type;
  /** @serial size of column (for VARCHAR column only) */
  public int size;

  public boolean equals(ColumnProperties other) {
    if ((type!=other.type)||(size!=other.size))
      return false;
    return name.equalsIgnoreCase(other.name);
  }

  public int hashCode() {
    return (size<<8) ^ type ^ (name!=null?name.hashCode():0);
  }
};


/**
 * <p>Options to be applied to column data after it has been loaded.
 *
 * <p>Supported options include:<br></ul>
 *   <li>add</li>
 *   <li>multiply</li>
 *   <li>divide</li>
 *   <li>change to null</li>
 *   <li>filter for number</li>
 *   <li>Y2K fix (make 4-digit year)</li></ul>
 *
 * @author chris.studholme@utoronto.ca
 */
class ColumnOption implements Serializable {

  /** @serial option type (one of: a,d,m,n,F,Y) */
  public char type;
    
  /** @serial numeric value need by option */
  public double nval;
  public boolean nvalvalid=false;
  /** @serial string value need by option */
  public String sval=null;

  public ColumnOption(String value) {
    type = value.charAt(0);
    if (value.length()>1) {
      sval = value.substring(1);
      try {
	nval = new Double(sval).doubleValue();
	nvalvalid = true;
      }
      catch (Exception e) {
      }
    }
  }
    
  public int hashCode() {
    return (new Double(nval).hashCode())^type^(sval!=null?sval.hashCode():0);
  }
};


/**
 * <p>Properties of an individual column as it is read from the
 * data file.  Many rows of table data can be obtains from one
 * block of data from the input file.  This class describes the
 * format of one column within one row of data obtains from a block.
 * The class ColumnProperties describes the output format for all
 * rows of one column.  ColumnOptions are applied to each row within
 * a block independently.
 *
 * @author chris.studholme@utoronto.ca
 */
final class SourceProperties implements Serializable {
  /** @serial SQL type of input data */
  public int type;
  /** @serial size of input field */
  public int size;
  /** @serial offset of field within record */
  public int offset;
  /** real size */
  public transient int rsize;
  /** real offset */
  public transient int roffset;

  /** @serial options to apply to column */
  public ColumnOption options[];
    
  public int hashCode() {
    int result = type ^ (offset<<8) ^ (size<<16);
    if (options!=null)
      for (int i=0; i<options.length; ++i)
	if (options[i]!=null)
	  result ^= roll(options[i].hashCode(),i);
    return result;
  }
    
  public static final int roll(int i, int count) {
    return (i<<count)|(i>>(32-count));
  }
};


/**
 * <p>A row option is applied to one row within a data block.  Each
 * block of data read from the data file can be used to generate
 * multiple rows of data.  Subclasses of this abstract class represent
 * specific row options.
 *
 * @author chris.studholme@utoronto.ca
 */
abstract class RowOption implements Serializable {
  /**
   * Verify that a row can be extracted from a block of data.
   *
   * @param brecord block of data
   * @return true if a row can be extracted, false otherwise
   */
  public abstract boolean rowOK(byte[] brecord);

  /**
   * Verify that a row can be extracted from a block of data.
   *
   * @param srecord block of data
   * @return true if a row can be extracted, false otherwise
   */
  public abstract boolean rowOK(String srecord);
}


/**
 * <p>Option to skip the row if a specific ascii string is
 * found at a specific location within the block.
 *
 * @author chris.studholme@utoronto.ca
 */
final class RowOption_skip extends RowOption {
  /** @serial offset of data to check */
  public int offset=0;
  /** @serial text to look for */
  public String text=null;
  
  public RowOption_skip(String optionstring) throws DatabaseException {
    String value = null;
    int index;
    if ((index=optionstring.indexOf(','))>=0) {
      value = optionstring.substring(0,index).trim();
      optionstring = optionstring.substring(index+1).trim();
    }
    else {
      value=optionstring;
      optionstring=null;
    }
    if ((value==null)||(value.length()<1))
      throw new DatabaseException("number expected");
    offset=Integer.parseInt(value);

    if ((index=optionstring.indexOf(','))>=0) {
      value = optionstring.substring(0,index).trim();
      optionstring = optionstring.substring(index+1).trim();
    }
    else {
      value=optionstring;
      optionstring=null;
    }
    if ((value==null)||(value.length()<1))
      throw new DatabaseException("string expected");
    text=value;
  }

  public boolean rowOK(byte[] brecord) {
    return ! new String(brecord,offset,text.length()).equals(text);
  }
  public boolean rowOK(String srecord) {
    return ! srecord.regionMatches(offset,text,0,text.length());
  }

  public int hashCode() {
    return 0x10101010 ^ offset ^ (text!=null?text.hashCode():0);
  }
}


/**
 * <p>Option to accept the row if a specific ascii string is
 * found at a specific location within the block.
 *
 * @author chris.studholme@utoronto.ca
 */
final class RowOption_select extends RowOption {
  /** @serial offset of data to check */
  public int offset=0;
  /** @serial text to look for */
  public String text=null;
  
  public RowOption_select(String optionstring) throws DatabaseException {
    String value = null;
    int index;
    if ((index=optionstring.indexOf(','))>=0) {
      value = optionstring.substring(0,index).trim();
      optionstring = optionstring.substring(index+1).trim();
    }
    else {
      value=optionstring;
      optionstring=null;
    }
    if ((value==null)||(value.length()<1))
      throw new DatabaseException("number expected");
    offset=Integer.parseInt(value);

    if ((index=optionstring.indexOf(','))>=0) {
      value = optionstring.substring(0,index).trim();
      optionstring = optionstring.substring(index+1).trim();
    }
    else {
      value=optionstring;
      optionstring=null;
    }
    if ((value==null)||(value.length()<1))
      throw new DatabaseException("string expected");
    text=value;
  }

  public boolean rowOK(byte[] brecord) {
    return new String(brecord,offset,text.length()).equals(text);
  }
  public boolean rowOK(String srecord) {
    return srecord.regionMatches(offset,text,0,text.length());
  }

  public int hashCode() {
    return 0x20202020 ^ offset ^ (text!=null?text.hashCode():0);
  }
}


// row id object
class RowId implements Serializable {
  public int file;
  public long offset;
  public int source;

  public RowId(int file, long offset, int source) {
    this.file = file;
    this.offset = offset;
    this.source = source;
  }

  public String toString() {
    return "["+file+":"+offset+":"+source+"]";
  }

  public boolean equals(Object o) {
    if (!(o instanceof RowId)) return false;
    return file==((RowId)o).file && 
      offset==((RowId)o).offset && source==((RowId)o).source;
  }
};


/**
 * <p>A DatabaseTable that can represent any flat ascii file
 * as a table for ModSQL to query.  A table definition file
 * describes the format of the ascii file.
 *
 * @author chris.studholme@utoronto.ca
 */
public final class AsciiTable implements DatabaseTable {
  
  /** Version number of compiled table definition file. */
  private static final int binfileVersion=3;

  /** Filename extension applied to compiled table definition files. */
  private static final String compiledExtension = ".bin";

  /** 
   * Initial record size for variable length record files.  This record
   * size is automatically double every time it is exceeded.  It should
   * reach its required length after the first few blocks are read from
   * the data file.
   */
  private static final int startrecordsize = 256;

  /** List of data file to read data from. */
  private transient AsciiReader[] datastream=null;
  /** Total number of data files. */
  private transient int nfiles=0;

  /** File where current pointer is located. */
  private transient int filenum=-1;

  /** Size of fixed length records or -1 for variable length records. */
  private int recordsize;

  /** Delimiter for delimited files. */
  private String delimiter;

  /** Max number of delimited fields. */
  private transient int maxdelimiter;

  /** Offset of start of data within data files. */
  private int dataoffset;
    
  /** Properties of the table's columns. */
  private ColumnProperties columns[];
  /** Number of columns. */
  private int ncolumns;

  /** 
   * Properties of the table's source rows.  Each block of data read
   * from the data files can contain multiple rows of data, each of
   * which can contain multiple columns. */
  private SourceProperties sources[][];
  /** Number of rows per data block. */
  private int nsourcerows;
  /** 
   * Each row within a block can have multiple options applied to it
   * before column data is extracted from it.
   */
  private RowOption options[][];

  /** Buffer for holding blocks of data. */
  private transient byte[] record=null;

  /** Length of valid data within record. */
  private transient int recordvalid;

  /** Offset of current record within current file. */
  private transient long rowoffset=-1;

  /** Current row within the current record. */
  private transient int sourceid=0;

  /** Name of this table. */
  private transient String name;

  /** 
   * This table's signature.  -1 indicates that the signature has
   * not yet been calculated. 
   */
  private transient long tablesignature=-1;


  /**
   * Open existing table.
   *
   * @param tablename name of table
   * @param deffilename path to table definition file
   * @exception IOException if error occurs reading table definition
   * @exception DatabaseException if other error occurs
   */
  public AsciiTable(String tablename, String deffilename) 
    throws IOException,DatabaseException {
    name = tablename;
    ReadTableDefinition(deffilename);
  }

  /**
   * Opens existing table.
   *
   * @param tablename name of table
   * @param deffilename path to table definition file
   * @param datafilename path to first data file
   * @exception IOException if error reading files
   * @exception DatabaseException if other error occurs
   */
  public AsciiTable(String tablename, String deffilename, String datafilename) 
    throws IOException,DatabaseException {
    name = tablename;
    ReadTableDefinition(deffilename);
    AddDataFile(datafilename);
  }

  /**
   * Close files.  Just calls close().
   */
  protected void finalize() {
    close();
  }

  /**
   * Add a new data file to the table.  Data files ending with '.gz' are
   * assumed to be GZIP compressed files.
   *
   * @param datafilename path to data file
   * @exception IOException if error reading data file
   * @exception DatabaseException if other error occurs
   */
  public void AddDataFile(String datafilename) 
    throws IOException,DatabaseException {
    if (datastream==null) 
      datastream = new AsciiReader[1];
    if (nfiles>=datastream.length) {
      AsciiReader[] newdatastream = new AsciiReader[datastream.length*2];
      System.arraycopy(datastream,0,newdatastream,0,nfiles);
      datastream=newdatastream;
    }
    datastream[nfiles] = new AsciiReader(datafilename,
					 datafilename.endsWith(".gz"));
    ++nfiles;
    beforeFirst();
  }

  /**
   * Read the table definition file.  First an attempt is made to read
   * the compiled version of the file.  If this file does not exist, is
   * unreadable, or is out of date, the raw table definition file is
   * read instead.  A new compiled version will be created (if possible).
   *
   * @param filename path to raw (uncompiled) table definition file
   * @exception IOException if error reading file occurs
   * @exception DatabaseException if other error occurs
   */
  protected void ReadTableDefinition(String filename) 
    throws DatabaseException,IOException {
    File rawfile = new File(filename);
    File compiledfile = new File(filename+compiledExtension);

    if (compiledfile.canRead()&&
	(compiledfile.lastModified()>rawfile.lastModified())) {
      // attempt to read compiled file
      if (ReadCompiledFile(compiledfile)) {
	//	java.lang.System.err.println("AsciiDatabase.AsciiTable: compiled file '"+compiledfile.toString()+"' read");
	return;
      }
      java.lang.System.err.println("AsciiDatabase.AsciiTable: failed to read "+
				   compiledfile.toString());
    }

    ReadRawTableDefinition(rawfile);
    WriteCompiledFile(compiledfile);
  }

  /**
   * Read the compiled (binary) version of the table definition file.
   *
   * @param filename File object pointing to binary file
   * @return true if the file was successfully read, false otherwise
   */
  private boolean ReadCompiledFile(File filename) {
    try {
      // open file
      FileInputStream filestream = new FileInputStream(filename);

      if (filestream.read()!=binfileVersion)
	return false;
      boolean compressed = (filestream.read()==1);

      ObjectInputStream in = 
	new ObjectInputStream(compressed ? 
			      (InputStream)new GZIPInputStream(filestream)
			      : (InputStream)new BufferedInputStream(filestream));
      
      // read data
      recordsize  = in.readInt();
      dataoffset  = in.readInt();
      ncolumns    = in.readInt();
      nsourcerows = in.readInt();
      delimiter = in.readUTF();
      if (delimiter.length()==0)
	delimiter=null;
      columns = (ColumnProperties[])in.readObject();
      sources = (SourceProperties[][])in.readObject();
      options = (RowOption[][])in.readObject();
      if (delimiter!=null) {
	maxdelimiter=0;
	for (int i=0; i<nsourcerows; ++i)
	  for (int j=0; j<ncolumns; ++j)
	    if (maxdelimiter<sources[i][j].offset)
	      maxdelimiter=sources[i][j].offset;
      }
      
      in.close();
      filestream.close();
    }
    catch (Exception e) {
      return false;
    }
    return true;
  }

  /**
   * Attemp to write the compiled (binary) version of the table 
   * definition file.  No indication of failure is provided.
   *
   * @param filename File object point to binary file
   */
  private void WriteCompiledFile(File filename) {
    // attempt to write binary version of tabledef file
    try {
      // reallocate columns
      SourceProperties newsource[] = new SourceProperties[ncolumns];
      ColumnProperties newcolumns[] = new ColumnProperties[ncolumns];
      System.arraycopy(sources[0],0,newsource,0,ncolumns);
      System.arraycopy(columns,0,newcolumns,0,ncolumns);
      sources[0]=newsource;
      columns=newcolumns;

      // reallocate sources
      SourceProperties newsources[][] = new SourceProperties[nsourcerows][];
      RowOption newoptions[][] = new RowOption[nsourcerows][];
      System.arraycopy(sources,0,newsources,0,nsourcerows);
      System.arraycopy(options,0,newoptions,0,nsourcerows);
      sources=newsources;
      options=newoptions;

      // open file
      FileOutputStream filestream = new FileOutputStream(filename);
      BufferedOutputStream buffer = new BufferedOutputStream(filestream);
      buffer.write(binfileVersion);
      buffer.write(0);

      ObjectOutputStream out = new ObjectOutputStream(buffer);
      
      // write data
      out.writeInt(recordsize);
      out.writeInt(dataoffset);
      out.writeInt(ncolumns);
      out.writeInt(nsourcerows);
      if (delimiter!=null)
	out.writeUTF(delimiter);
      else
	out.writeUTF("");
      out.writeObject(columns);
      out.writeObject(sources);
      out.writeObject(options);
      out.flush();
      out.close();
      buffer.flush();
      buffer.close();
      filestream.close();
    }
    catch (Exception e) {
    }
  }


  /**
   * Parse column type string.  This test is case insensitive.  Types
   * supported include: char, int, long, single, double, bool.
   *
   * @param type string to parse
   * @return value from java.sql.Types
   */
  private int ParseType(String type) {
    if (type.equalsIgnoreCase("char"))
      return Types.VARCHAR;
    else if (type.equalsIgnoreCase("int"))
      return Types.INTEGER;
    else if (type.equalsIgnoreCase("long"))
      return Types.BIGINT;
    else if (type.equalsIgnoreCase("single"))
      return Types.REAL;
    else if (type.equalsIgnoreCase("double"))
      return Types.DOUBLE;
    else if (type.equalsIgnoreCase("bool"))
      return Types.BIT;
    //    else if (type.equalsIgnoreCase(""))
    //      return ;
    return Types.OTHER;
  }


  /**
   * Parse a row option.
   *
   * @param optionstring String to parse
   * @return RowOption object
   * @exception DatabaseException if an error occurs
   */
  private RowOption ParseRowOption(String optionstring) 
    throws DatabaseException {
    // options are of the form option=type[,type specifics,...]
    String type = null;
    int index;
    if ((index=optionstring.indexOf(','))>=0) {
      type = optionstring.substring(0,index).trim();
      optionstring = optionstring.substring(index+1).trim();
    }
    else {
      type=optionstring;
      optionstring=null;
    }

    if (type==null)
      return null;

    if (type.equals("skip"))
      return new RowOption_skip(optionstring);

    else if (type.equals("select"))
      return new RowOption_select(optionstring);

    //    else if (type.equals("skip"))
    //      return new RowOption_Skip(optionstring);

    return null;
  }

  /**
   * Parse column options.  If optionstring starts with '#',
   * it is ignored.
   *
   * @param optionstring String to parse
   * @return ColumnOption[] array
   * @exception DatabaseException if error occurs 
   */
  private ColumnOption[] ParseColumnOptions(String optionstring) 
    throws DatabaseException {

    // options are of the form [char][value],...
    optionstring = optionstring.trim();
    if (optionstring.startsWith("#"))
      return null;

    Vector options = new Vector();

    while (optionstring!=null) {

      String value = null;
      int index;
      if ((index=optionstring.indexOf(','))>=0) {
	value = optionstring.substring(0,index).trim();
	optionstring = optionstring.substring(index+1).trim();
      }
      else {
	value=optionstring;
	optionstring=null;
      }

      if ((value!=null)&&(value.length()>0)) {
	ColumnOption option = new ColumnOption(value);
	options.addElement(option);
      }
    }

    ColumnOption[] result = null;
    if (options.size()>0) {
      result = new ColumnOption[options.size()];
      for (int i=0; i<options.size(); ++i)
	result[i] = (ColumnOption)(options.elementAt(i));
    }

    return result;
  }


  /**
   * Parse an entire column definition.  Column definition is a line
   * of text with the following columns:<pre>
   *   source_type source_length source_offset column_type column_name options</pre>
   * <p>Obviously, options are optional.  Any whitespace can seperate the 
   * columns.
   *
   * @param column ColumnProperties object to set
   * @param source SourceProperties object to set
   * @param columndef ascii line to parse
   * @param offset to be added to source_offset before storing in SourceProperties
   * @exception IOException if an error occurs
   * @exception DatabaseException if an error occurs
   */
  private void ParseColumnDef(ColumnProperties column, 
			      SourceProperties source,
			      String columndef, int offset) 
    throws IOException, DatabaseException {

    columndef = columndef.replace('\t',' ');
    columndef = columndef.trim();
    String value = null;
    int index;

    // source type
    if ((columndef!=null)&&((index=columndef.indexOf(' '))>0)) {
      value = columndef.substring(0,index).trim();
      columndef = columndef.substring(index).trim();
    }
    else {
      value=columndef;
      columndef=null;
    }
    if ((value==null)||(value.length()<1))
      throw new IOException("string expected");
    source.type=ParseType(value);
    if (source.type==Types.OTHER)
      throw new IOException("invalid source type");

    // source length
    if ((columndef!=null)&&((index=columndef.indexOf(' '))>0)) {
      value = columndef.substring(0,index).trim();
      columndef = columndef.substring(index).trim();
    }
    else {
      value=columndef;
      columndef=null;
    }
    if ((value==null)||(value.length()<1))
      throw new IOException("number expected");
    source.size=Integer.parseInt(value);
    column.size=source.size;

    // source offset
    if ((columndef!=null)&&((index=columndef.indexOf(' '))>0)) {
      value = columndef.substring(0,index).trim();
      columndef = columndef.substring(index).trim();
    }
    else {
      value=columndef;
      columndef=null;
    }
    if ((value==null)||(value.length()<1))
      throw new IOException("number expected");
    source.offset=offset+Integer.parseInt(value);

    // column type
    if ((columndef!=null)&&((index=columndef.indexOf(' '))>0)) {
      value = columndef.substring(0,index).trim();
      columndef = columndef.substring(index).trim();
    }
    else {
      value=columndef;
      columndef=null;
    }
    if ((value==null)||(value.length()<1))
      throw new IOException("string expected");
    column.type=ParseType(value);
    if (column.type==Types.OTHER)
      throw new IOException("invalid column type");
    if (column.type!=Types.VARCHAR)
      column.size=0;

    // column name
    if ((columndef!=null)&&((index=columndef.indexOf(' '))>0)) {
      value = columndef.substring(0,index).trim();
      columndef = columndef.substring(index).trim();
    }
    else {
      value=columndef;
      columndef=null;
    }
    if ((value==null)||(value.length()<1))
      throw new IOException("string expected");
    column.name=value;
    
    // source options
    if ((columndef!=null)&&((index=columndef.indexOf(' '))>0)) {
      value = columndef.substring(0,index).trim();
      columndef = columndef.substring(index).trim();
    }
    else {
      value=columndef;
      columndef=null;
    }
    if ((value!=null)&&(value.length()>0))
      source.options = ParseColumnOptions(value);
  }


  /**
   * Read a raw (uncompressed) table definition file.
   *
   * @param filename File object pointing to raw file
   * @exception DatabaseException if an error occurs
   */
  private void ReadRawTableDefinition(File filename) 
    throws DatabaseException {
    try {
      RCFile deffile = new RCFile(filename);
      String rs = deffile.ReadCharValue("recordsize");
      if (rs==null)
	throw new DatabaseException("AsciiTable requires recordsize parameter");
      recordsize = Integer.parseInt(rs);

      delimiter = deffile.ReadCharValue("delimiter");
      if (delimiter!=null) {
	delimiter=delimiter.trim();
	if (delimiter.length()==0)
	  delimiter=null;
	else {
	  if (delimiter.equals("\\t"))
	    delimiter="\t";
	}
      }
      
      String of = deffile.ReadCharValue("offset");
      dataoffset = (of==null ? 0 : Integer.parseInt(of));

      int colsallocated = 1;
      int rowsallocated = 1;
      columns = new ColumnProperties[1];
      sources = new SourceProperties[1][];
      sources[0] = new SourceProperties[1];
      options = new RowOption[1][];
      nsourcerows=0;
      ncolumns=0;

      // read rows
      deffile.ResetFile();
      String header;
      while (((header=deffile.FindNextHeader())!=null)&&
	     (header.equalsIgnoreCase("row"))) {

	// allocate array for next row
	if (nsourcerows>=rowsallocated) {
	  rowsallocated *= 2;
	  SourceProperties newsources[][] = 
	    new SourceProperties[rowsallocated][];
	  RowOption newoptions[][] = new RowOption[rowsallocated][];
	  System.arraycopy(sources,0,newsources,0,nsourcerows);
	  System.arraycopy(options,0,newoptions,0,nsourcerows);
	  sources=newsources;
	  options=newoptions;
	}
	if (nsourcerows>0)
	  sources[nsourcerows] = new SourceProperties[ncolumns];
	options[nsourcerows] = null;

	// read columns
	String value;
	int icolumn=0;
	int offset=0;
	int noptions=0;
	int optionsallocated=0;

	while ((value=deffile.ReadNextCharValue())!=null) {
	  if (deffile.LastProperty().equalsIgnoreCase("offset")) 
	    offset=Integer.parseInt(value);

	  else if (deffile.LastProperty().equalsIgnoreCase("option")) {
	    if (options[nsourcerows]==null) {
	      options[nsourcerows] = new RowOption[1];
	      optionsallocated=1;
	    }
	    if (optionsallocated<=noptions) {
	      RowOption newoptions[] = new RowOption[optionsallocated*2];
	      for (int i=0; i<optionsallocated; ++i) 
		newoptions[i]=options[nsourcerows][i];
	      options[nsourcerows]=newoptions;
	      optionsallocated*=2;
	    }
	    options[nsourcerows][noptions] = ParseRowOption(value);
	    if (options[nsourcerows][noptions]!=null)
	      ++noptions;
	    else
	      throw new DatabaseException("invalid row option");
	  }
	  
	  else if (deffile.LastProperty().equalsIgnoreCase("column")) {
	    ColumnProperties column = new ColumnProperties();
	    SourceProperties source = new SourceProperties();
	    ParseColumnDef(column,source,value,offset);
	    
	    if (nsourcerows==0) {
	      if (ncolumns>=colsallocated) {
		colsallocated *= 2;
		SourceProperties newsources[] = 
		  new SourceProperties[colsallocated];
		ColumnProperties newcolumns[] = 
		  new ColumnProperties[colsallocated];
		System.arraycopy(sources[0],0,newsources,0,ncolumns);
		System.arraycopy(columns,0,newcolumns,0,ncolumns);
		sources[0]=newsources;
		columns=newcolumns;
	      }
	      sources[0][ncolumns]=source;
	      columns[ncolumns++]=column;
	    }
	    
	    else {
	      if (icolumn>=ncolumns)
		throw new DatabaseException("each row must have same number of columns");
	      if (!column.equals(columns[icolumn]))
		throw new DatabaseException("columns in each row must be identical");
	      sources[nsourcerows][icolumn] = source;
	    }
	    
	    ++icolumn;
	  }
	}
	++nsourcerows;
      }

      if (delimiter!=null) {
	maxdelimiter=0;
	for (int i=0; i<nsourcerows; ++i)
	  for (int j=0; j<ncolumns; ++j)
	    if (maxdelimiter<sources[i][j].offset)
	      maxdelimiter=sources[i][j].offset;
      }

      deffile.close();
    }
    catch (Exception e) {
      if (e instanceof DatabaseException)
	throw (DatabaseException)e;
      java.lang.System.err.println(e.toString());
      throw new DatabaseException("error parsing table definition file");
    }

  }


  //======================================================================
  // Methods for manipulating table
  //======================================================================

  /**
   * Open index of specified column.  Indexes are not currently supported.
   *
   * @param column index of column to find index for
   * @return DatabaseTable object for index, or null if column is not indexed
   */
  public DatabaseIndex openIndex(int column) {
    return null;
  }

  /**
   * Close the table.  Closes all data files.
   */
  public void close() {
    record=null;
    for (int i=0; i<nfiles; ++i) {
      datastream[i].close();
      datastream[i]=null;
    }
    nfiles=0;
  }


  //======================================================================
  // Methods for accessing table metadata
  //======================================================================

  /**
   * Is a particular column indexed?  No.  Indexes aren't supported yet.
   *
   * @param column index of column that should be indexed
   * @return true if the column is indexed, false otherwise
   */
  public boolean isIndexAvailable(int column) {
    return false;
  }

  /**
   * Is the table read only?  Yes.  Read/write tables aren't supported yet.
   *
   * @return true if the table is read only, false otherwise
   */
  public boolean isReadOnly() {
    return true;
  }

  /**
   * Get table name.  This is just the name passed to the constructor.
   *
   * @return table name
   */
  public String getTableName() {
    return name;
  }

  /**
   * Get count of columns.
   *
   * @return number of columns in table
   */
  public int getColumnCount() {
    return ncolumns;
  }

  /**
   * Get size of table.  This size should not be considered accurate.  For
   * GZIP compressed data files, the size returned is the total size of the
   * compressed files.  It is not possible to determine the size of the
   * uncompressed data.
   *
   * @return total size of data files
   */
  public long getTableSize() {
    long size=0;
    for (int i=0; i<nfiles; ++i)
      size+=datastream[i].length();
    return size;
  }

  /**
   * Calculate a unique signature for this table.  This signature should
   * depend on the following:<br><ul>
   *   <li>int recordsize</li>
   *   <li>int dataoffset</li>
   *
   *   <li>ColumnProperties columns[]</li>
   *   <li>SourceProperties sources[][]</li>
   *   <li>RowOption options[][]</li>
   *   <li>int ncolumns</li>
   *   <li>int nsourcerows</li>
   *
   *   <li>lengths and order of data files</li></ul>
   *   
   * <p>Any change in the table that would invalidate an index should
   * result in a change of this signature.
   *
   * @return table signature
   */
  protected long calculateTableSignature() {
    Random gen = new Random((((long)dataoffset)<<32)+recordsize);
    long hash = gen.nextLong()^((((long)ncolumns)<<32)+nsourcerows);
    for (int i=0; i<ncolumns; ++i) {
      hash ^= gen.nextLong()*columns[i].hashCode();
      for (int j=0; j<nsourcerows; ++j) 
	hash ^= gen.nextLong()*sources[j][i].hashCode();
    }
    for (int j=0; j<nsourcerows; ++j)
      if (options[j]!=null)
	for (int i=0; i<options[j].length; ++i)
	  if (options[j][i]!=null)
	    hash ^= gen.nextLong()*options[j][i].hashCode();
    for (int i=0; i<nfiles; ++i)
      hash ^= gen.nextLong()*datastream[i].length();
    return hash;
  }

  /**
   * Get signature for table.  In very rare circumstances, table signature
   * will be recalculated everytime this function is called (if the signature
   * just happens to be -1), otherwise the signature is cached.
   *
   * @return table signature
   */
  public long getTableSignature() {
    if (tablesignature==-1)
      tablesignature=calculateTableSignature();
    return tablesignature;
  }


  public long getRowCount() {
    return -1;
  }


  //======================================================================
  // Methods for accessing column metadata
  //======================================================================

  /**
   * Find a column by name.
   *
   * @param name name of column to search for
   * @return index of column, -1 if the column was not found
   */
  public int findColumn(String name) {
    for (int i=0; i<ncolumns; ++i)
      if (columns[i].name.equalsIgnoreCase(name))
	return i+1;
    return -1;
  }

  /**
   * Get ideal display width for column.  Only applied to VARCHAR columns.
   *
   * @param column index of column
   * @return width in characters
   */
  public int getColumnDisplaySize(int column) {
    if ((column>0)&&(column<=ncolumns))
      return columns[column-1].size;
    return 0;
  }
  
  /**
   * Get an appropriate lable for the column.
   *
   * @param column index of column
   * @return label to be used for column
   */
  public String getColumnLabel(int column) {
    if ((column>0)&&(column<=ncolumns))
      return columns[column-1].name;
    return null;
  }

  /**
   * Get name of column.
   *
   * @param column index of column
   * @return name of column
   */
  public String getColumnName(int column) {
    if ((column>0)&&(column<=ncolumns))
      return columns[column-1].name;
    return null;
  }
  
  /**
   * Get SQL type of column.
   *
   * @param column index of column
   * @return value from java.sql.Types
   */
  public int getColumnType(int column) {
    if ((column>0)&&(column<=ncolumns))
      return columns[column-1].type;
    return Types.NULL;
  }

  /**
   * Add a column to the database.  Not currently supported.
   *
   * @param name name of new column
   * @param type SQL type of new column
   * @exception DatabaseException in case of error
   */
  public void addColumn(String name, int type, int maxlen)
    throws DatabaseException {
    throw new DatabaseException("cannot add column");
  }
  

  //======================================================================
  // Methods for changing current row
  //======================================================================
    
  /** Delimit fields.
   */
  private void delimitFields() {
    if (delimiter==null)
      for (int i=0; i<ncolumns; ++i) {
	sources[sourceid][i].roffset=sources[sourceid][i].offset;
	sources[sourceid][i].rsize=sources[sourceid][i].size;
      }
    else {
      String rec = new String(record,0,recordvalid);
      int dl=delimiter.length();
      int starts[] = new int[maxdelimiter+1];
      int lens[] = new int[maxdelimiter+1];
      int pos=0;
      int i=1;
      starts[0]=0;
      while ((i<=maxdelimiter)&&(pos<recordvalid)) {
	pos = rec.indexOf(delimiter,pos);
	if (pos==-1)
	  pos=recordvalid;
	lens[i-1]=pos-starts[i-1];
	pos += dl;
	starts[i++]=pos;
      }
      while (i<=maxdelimiter) {
	lens[i-1]=recordvalid-starts[i-1];
	starts[i++]=recordvalid;
      }
      if (pos<recordvalid) {
	pos = rec.indexOf(delimiter,pos);
	if (pos==-1)
	  pos=recordvalid;
      }
      lens[maxdelimiter]=pos-starts[maxdelimiter];

      for (i=0; i<ncolumns; ++i) {
	sources[sourceid][i].roffset=starts[sources[sourceid][i].offset];
	sources[sourceid][i].rsize=lens[sources[sourceid][i].offset];
	if (sources[sourceid][i].rsize>sources[sourceid][i].size)
	  sources[sourceid][i].rsize=sources[sourceid][i].size;
      }
    }
  }

  /**
   * Advance to next row.
   *
   * @return true if next row is valid, false otherwise
   * @exception DatabaseException if an error occurs
   */
  public boolean next() throws DatabaseException {
    while (true) {

      if (++sourceid<nsourcerows)
	delimitFields();

      else {
	// read new record from data file
	sourceid=0;

	try {
	  rowoffset = datastream[filenum].getFilePointer();

	  if (recordsize>0) {
	    // fixed length records
	    if (datastream[filenum].read(record)!=recordsize) {
	      // start next file
	      if (++filenum<nfiles) {
		datastream[filenum].seek(dataoffset);
		sourceid = nsourcerows;
		continue;
	      }
	      else return false;
	    }
	    recordvalid=recordsize;
	    delimitFields();
	  } 
	  
	  else {
	    // variable length records
	    recordvalid = datastream[filenum].readToEOL(record);
	    
	    // record is incomplete
	    while ((recordvalid==record.length)&&
		   (record[recordvalid-1]!='\r')&&
		   (record[recordvalid-1]!='\n')) {
	      byte newrecord[] = new byte[record.length*2];
	      for (int i=0; i<record.length; ++i)
		newrecord[i]=record[i];
	      recordvalid = record.length + 
		datastream[filenum].readToEOL(newrecord,record.length,
					      record.length);
	      record=newrecord;
	    }
	    
	    if (recordvalid<=0) {
	      // start next file
	      if (++filenum<nfiles) {
		datastream[filenum].seek(dataoffset);
		sourceid = nsourcerows;
		continue;
	      }
	      else return false;
	    }

	    record[--recordvalid]=0;
	    delimitFields();
	  }
	}
	catch (Exception e) {
	  throw new DatabaseException("unable to read row (was "+
				      e.toString()+")");
	}

      } 
      
      // verify that this row is valid
      if (options[sourceid]!=null) {
	boolean skip=false;
	for (int i=0; i<options[sourceid].length; ++i)
	  if (options[sourceid][i]==null)
	    break;
	  else
	    if (!options[sourceid][i].rowOK(record)) {
	      skip=true;
	      break;
	    }
	if (skip)
	  continue;
      }
      
      return true;
    }
  }

  /**
   * Find next row containing the specified data.
   *
   * @param columnIndex index of column to contain data
   * @param data data to search for
   * @return true if the desired row was found, false otherwise
   * @exception DatabaseException if an error occured
   */
  public boolean findNext(int columnIndex, Object data) 
    throws DatabaseException {
    while (next()) {
      Object o = getObject(columnIndex);
      if (data==null) {
        if (o==null)
          return true;
      }
      else if (data.equals(o))
        return true;
    }
    return false;
  }

  /**
   * Delete the current row.
   *
   * @exception DatabaseException if an error occured
   */
  public void deleteRow() throws DatabaseException {
    throw new DatabaseException ("cannot delete row");
  }

  /**
   * Add a new row.
   *
   * @exception DatabaseException if an error occured.
   */
  public void addRow() throws DatabaseException {
    throw new DatabaseException ("cannot add row");
  }

  /**
   * Is before first row?
   *
   * @return true if pointer is before the first row, false otherwise
   */
  public boolean isBeforeFirst() {
    return (rowoffset<0);
  }

  /**
   * Is after last row?
   *
   * @return true if pointer is after the last row, false otherwise
   */
  public boolean isAfterLast() {
    return (filenum>=nfiles);
  }

  /**
   * Reset the table to before the beginning of the specified data file.
   *
   * <p>Note that this does not mean that if you reset to before the
   * beggining of file 2, that the pointer is actually at the end of
   * file 1.
   *
   * @param file to position pointer before
   * @exception DatabaseException if an error occurs
   */
  private void ResetTable(int file) throws DatabaseException {
    try {
      filenum=file;
      datastream[filenum].seek(dataoffset);
      record = (recordsize>0 ? new byte[recordsize] : 
		new byte[startrecordsize]);
      rowoffset = -1;
      sourceid = nsourcerows;
    }
    catch (Exception e) {
      if (e instanceof FileNotFoundException)
	throw new DatabaseException("table data file not found");
      throw new DatabaseException("unknown error (was "+e.toString()+")");
    }
  }

  /**
   * Position pointer before first row.
   *
   * @exception DatabaseException if error occurs
   */
  public void beforeFirst() throws DatabaseException {
    ResetTable(0);
  }

  /**
   * Position pointer after last row.
   *
   * @exception DatabaseException if error occurs
   */
  public void afterLast() throws DatabaseException {
    filenum=nfiles;
  }
  
  /**
   * <p>Get the id of the current row.
   *
   * <p>The row id is of the form [offset][file][source] where
   * offset is most significant and source is least significant.  It
   * is not valid to compare row id's to determine if one row is
   * before or after another.  You can compare row id's to determine
   * if two rows are the same row.
   *
   * @return row id
   */
  public Object getRowId() {
    return new RowId(filenum,rowoffset,sourceid);
  }

  /**
   * Seek to an absolute row id.  This method only accepts row id's
   * as returned from getRowId().
   *
   * @param id row id to seek to
   * @return true if the row was found, false otherwise
   * @exception DatabaseException if an error occurs
   */
  public boolean absolute(Object id) throws DatabaseException {
    RowId rowid = (RowId)id;
    long offset = rowid.offset;

    int oldfile = filenum;
    filenum = rowid.file;

    if (offset<dataoffset)
      throw new DatabaseException("bad rowid "+id);

    if ((oldfile!=filenum)||(offset!=rowoffset)) {
      try {
	datastream[filenum].seek(offset);
      }
      catch (Exception e) {
	throw new DatabaseException("cannot seek within datafile (was "+
				    e.toString()+")");
      }
      sourceid = nsourcerows;
      if (!next())
	return false;
    }
    sourceid = rowid.source;
    return true;
  }


  //======================================================================
  // Methods for accessing column data
  //======================================================================

  /**
   * Add a double to a Number.  The result is an object of the same
   * type as n.
   *
   * @param n Number to add
   * @param d double to add to the Number
   * @return Number object of same class as n
   * @exception DatabaseException if n is an invalid type
   */
  private static Number Add(Number n, double d) throws DatabaseException {
    if (n instanceof Integer)
      return new Integer((int)(n.doubleValue()+d));
    if (n instanceof Float)
      return new Float(n.doubleValue()+d);
    if (n instanceof Long)
      return new Long((long)(n.doubleValue()+d));
    if (n instanceof Double)
      return new Double(n.doubleValue()+d);
    throw new DatabaseException("invalid instance of Number");
  }


  /**
   * Multiply a double to a Number.  The result is an object of the
   * same class as n.
   *
   * @param n Number to multiply
   * @param d double to multiply to the Number
   * @return Number object of same class as n
   * @exception DatabaseException if n is an invalid type
   */
  private static Number Multiply(Number n, double d) 
    throws DatabaseException {
    if (n instanceof Integer)
      return new Integer((int)(n.doubleValue()*d));
    if (n instanceof Float)
      return new Float(n.doubleValue()*d);
    if (n instanceof Long)
      return new Long((long)(n.doubleValue()*d));
    if (n instanceof Double)
      return new Double(n.doubleValue()*d);
    throw new DatabaseException("invalid instance of Number");
  }


  /**
   * Divide a Number by a double.  The result is an object of the
   * same class as n.
   *
   * @param n Number to divide
   * @param d double to divide Number by
   * @return Number object of same class as n
   * @exception DatabaseException if n is an invalid type
   */
  private static Number Divide(Number n, double d) 
    throws DatabaseException {
    if (n instanceof Integer)
      return new Integer((int)(n.doubleValue()/d));
    if (n instanceof Float)
      return new Float(n.doubleValue()/d);
    if (n instanceof Long)
      return new Long((long)(n.doubleValue()/d));
    if (n instanceof Double)
      return new Double(n.doubleValue()/d);
    throw new DatabaseException("invalid instance of Number");
  }


  /**
   * Convert 2-digit year to 4-digit year.  
   *
   * @param n date to adjust
   * @param d maximum date to consider as 2000+date
   * @return Integer object representing new date
   * @exception DatabaseException if n is an invalid type
   */
  private static Number Y2K(Number n, double d)
    throws DatabaseException {
    int date = n.intValue();
    if (10000<=d && d<1000000) {  // year, month and day
      if (date<=d)
	return new Integer(date+20000000);
      else
	return new Integer(date+19000000);
    }
    else if (100<=d && d<10000) { // year and month
      if (date<=d)
	return new Integer(date+200000);
      else
	return new Integer(date+190000);
    }
    else if (1<=d && d<100) { // year only
      if (date<=d)
	return new Integer(date+2000);
      else
	return new Integer(date+1900);
    }
    else
      throw new DatabaseException("invalid date threshold");
  }

  /**
   * Filter string of a number.  This method does two things:<br><ol>
   *  <li>filter out non-numeric characters</li>
   *  <li>if a + or - is on the end of the string, move it to the front</li></ol>
   *
   * <p>Note: 2 is a hack to solve problems with our ROB database
   *
   * @param s string to filter
   * @return filtered string
   * @exception DatabaseException if an error occurs
   */
  private static String FilterForNumber(String s) throws DatabaseException {
    // filter out 'bad' characters
    for (int i=0; i<s.length(); ++i) {
      char c = s.charAt(i);
      if ((c!='.')&&(c!='e')&&(c!='E')&&(c!='-')&&((c<'0')||(c>'9'))) {
	s = (i>0?s.substring(0,i):"") + (i<s.length()-1?s.substring(i+1):"");
	--i;
      }
    }

    // check for '-' at end of string
    if (s.endsWith("-"))
      s = "-" + s.substring(0,s.length()-1);

    return s;
  }
  
  /**
   * Read column data.
   *
   * @param column index of column to read
   * @return Object representing column data
   * @exception DatabaseException if an error occurs
   */
  public Object getObject(int column) throws DatabaseException {
    if (--column<0)
      throw new DatabaseException("column not found");
    Object value = null;

    if (record==null) 
      throw new DatabaseException("row is not valid");

    // make sure column data is within valid record data
    int offset = sources[sourceid][column].roffset;
    int size = sources[sourceid][column].rsize;
    if (offset>=recordvalid)
      return null;
    if (offset+size>recordvalid)
      size = recordvalid-offset;
    
    switch (sources[sourceid][column].type) {
    case Types.VARCHAR:

      String s = new String(record,offset,size).trim();

      if ((sources[sourceid][column].options!=null)&&
	  (sources[sourceid][column].options[0]!=null)&&
	  (sources[sourceid][column].options[0].type=='F'))
	s = FilterForNumber(s);

      switch (columns[column].type) {
      case Types.CHAR:
      case Types.VARCHAR:
      case Types.LONGVARCHAR:
	value = s;
	break;

      case Types.TINYINT:
      case Types.SMALLINT:
      case Types.INTEGER:
	value = s.length()>0 ? new Integer(s) : null;
	break;

      case Types.BIGINT:
	value = s.length()>0 ? new Long(s) : null;
	break;

      case Types.REAL:
	value = s.length()>0 ? new Float(s) : null;
	break;

      case Types.FLOAT:
      case Types.DOUBLE:
	value = s.length()>0 ? new Double(s) : null;
	break;

      case Types.BIT:
	if (s.length()<=0) 
	  value=null;
	else
	  value = new Boolean(s.equals("1")||
			      s.equalsIgnoreCase("yes")||
			      s.equalsIgnoreCase("true"));
	break;

      default:
	throw new DatabaseException("invalid column type");
      }
      break;

    default:
      throw new DatabaseException("invalid source type");
    }

    if (value==null)
      return null;

    // apply column options
    if (sources[sourceid][column].options!=null)
      for (int i=0; i<sources[sourceid][column].options.length; ++i) {
	ColumnOption option = sources[sourceid][column].options[i];
	switch (option.type) {
	case 'a':
	  if (value instanceof Number)
	    value = Add((Number)value,option.nval);
	  else
	    throw new DatabaseException("add option only works with numbers");
	  break;

	case 'd':
	  if (value instanceof Number)
	    value = Divide((Number)value,option.nval);
	  else
	    throw new DatabaseException("divide option only works with numbers");
	  break;

	case 'm':
	  if (value instanceof Number)
	    value = Multiply((Number)value,option.nval);
	  else
	    throw new DatabaseException("multiply option only works with numbers");
	  break;
 
	case 'Y':   //Y2K fix
	  if (value instanceof Number)
	    value = Y2K((Number)value,option.nval);
	  else
	    throw new DatabaseException("Y2K option only works with numbers");
	  break;
 
	case 'n':
	  if (option.nvalvalid && value instanceof Number) {
	    if (((Number)value).floatValue()==(float)option.nval) 
	      return null;
	  }
	  else {
	    if (option.sval.equals(value.toString()))
	      return null;
	  }
	  break;

	case 'F':
	  // handled above
	  break;

	default:
	  throw new DatabaseException("unrecognized column option: "+
				      option.type);
	}
      }

    return value;
  }

  /**
   * Change column data.
   *
   * @param columnIndex index of column to change
   * @param x Object to set column to
   * @exception DatabaseException if an error occurs
   */
  public void updateObject(int columnIndex, Object x) 
    throws DatabaseException {
    throw new DatabaseException("cannot update row");
  }
  public void commitUpdates() throws DatabaseException {
    throw new DatabaseException("cannot update row");
  }
};