/*
 * @(#)JavaCGIBridge.java 2.00 07/18/98
 *
 * Copyright Info: This class was written by Gunther Birznieks
 * gunther@clark.net having been inspired by countless other
 * software developers.
 *
 * Version 1.00: 08/10/97
 * Version 2.00: 07/19/98
 *
 * The second version of this software was written for
 * independent study under the guidance of Professor 
 * Marty Hall as part of the Johns Hopkins University 
 * Computer Science Master's Degree program.
 *
 * Feel free to copy, cite, reference, sample, borrow, resell
 * or plagiarize the contents.  However, if you don't mind,
 * please let me know where it goes so that I can at least
 * watch and take part in the development of the memes. Information
 * wants to be free; support public domain freeware.  Donations are
 * appreciated and will be spent on further upgrades and other public
 * domain programs.
 *
 */

package com.extropia.net;

import java.util.Hashtable;
import java.util.Enumeration;
import java.util.Vector;
import java.util.Observable;

import java.net.URL;
import java.net.URLConnection;
import java.net.URLEncoder;
import java.net.MalformedURLException;

import java.io.ObjectOutputStream;
import java.io.ByteArrayOutputStream;
import java.io.InputStream;
import java.io.DataOutputStream;
import java.io.ByteArrayInputStream;
import java.io.ObjectInputStream;
import java.io.IOException;
import java.io.StreamCorruptedException;
import java.io.OptionalDataException;

import java.applet.AppletContext;

/**
 * This is a class that POSTS and GETS data from URLs using various
 * helper methods.  This class also provides the capability of
 * timing out if the connection takes too long to transfer data
 * by using threads to monitor whether the data is taking too long
 * to transfer (implements Runnable).
 * <P>
 * Helper methods allow you to set up form variables, get setup
 * information over a URL, get raw or pre-parsed HTML data, and more.
 * <P>
 * The getParsedData method relies on instance variables which
 * tell the parser where the data begins and ends (top and bottom
 * separators respectively).   The field and row separators inform
 * the parser where records and fields end.
 * <P><BLOCKQUOTE>
 * The default start of data separator is &LT;!--start of data-->\n
 * <P>
 * The default end of data separator is &LT;!--end of data-->\n
 * <P>
 * The default field separator is | (pipe).
 * <P>
 * The default record separator is \n (newline).
 * <P>
 * If the data you are sending is likely to contain special characters
 * such as newlines in database fields or | symbols, you can expand
 * the record and field separators to be more restrictive.  For example,
 * you might consider making the field seperator ~|~ (tilde + pipe + tilde) 
 * if you think a stray pipe might appear somewhere.
 * <P>
 * Another strategy is to follow a MIME like strategy where you make
 * very long separators that are very unlikely to appear randomly in
 * binary or large database field data. An example separator string
 * might be "__CabbageNeck_Peter_Joe_Erik_Mark_Anthony_David_987XxX". 
 * However, be aware that although longer separator strings provide
 * more security against random stray characters matching, they will
 * also increase the parsing time.
 * </BLOCKQUOTE>
 *
 * @version     2.00, 18 Jul 1998
 * @author      Gunther Birznieks
 * @author      Marty Hall
 *
 */
public class JavaCGIBridge extends Observable implements Runnable {

    // PUBLIC INSTANCE VARIABLES

    /**
     * SETUP indicates that the processing status is
     * in setup mode. This means that no communications
     * have been established yet. In otherwords, the programmer
     * is still setting up defaults.  
     *
     * @see #getStatus
     * @see #setStatus
     */
    public static final int SETUP = 1;

    /**
     * PROCESSING indicates that the JavaCGIBridge 
     * object is in the middle of processing a request.
     *
     * @see #getStatus
     * @see #setStatus
     */
    public static final int PROCESSING = 2;

    /**
     * IDLE indicates that the JavaCGIBridge object
     * has finished processing and can move on to do
     * something else. Usually this means that the
     * JavaCGIBridgePool object can give this instance
     * to another consumer now that it is done with 
     * the request.
     *
     * @see #getStatus
     * @see #setStatus
     */
    public static final int IDLE = 3;

    // PROTECTED INSTANCE VARIABLES

    /**
     * This is the vector of vectors of parsed
     * data. It is protected so that fetchNextRecord() can
     * peep it in the JavaCGIBridgeExtension class.
     */
    protected Vector _vectorParsedData = null;

    /**
     * This is the actual CGI Timeout value in milliseconds for the
     * instantiated object. Notice that the default value is a static
     * class variable that applies across all instances of this class.
     * This variable, on the other hand, is the actual value the object
     * uses.
     * <P>
     * It is protected so that the other JavaCGIBridge-type classes
     * can also detect whether a JavaCGIBridge object has timed out.
     */
    protected int _threadTimeOut = 10000;

    /**
     * This is a flag indicating whether the URL data was
     * retrieved inside the thread or not. It is protected
     * so that other JavaCGIBridge-type classes can 
     * figure out if a JavaCGIBridge object has timed out.
     */
    protected boolean _threadCompleted = true;

    /**
     * This lets us know at what point fetchNextRecord()
     * needs to read the next Vector.   This variable is
     * located here so that it can be manipulated by the
     * communications thread as it parses Vector data
     * for consumption by fetchNextRecord().
     */
    protected int _fetchCount = 0;

    /**
     * This variable contains the base time that
     * the thread last received valid data.  The base
     * time basically tells the object how much time
     * has past between successful communications when
     * compared with System.currentTimeMillis(). It is
     * protected so that other JavaCGIBridge-type objects
     * can figure out whether a JavaCGIBridge object has
     * timed out.
     */
    protected long _threadBaseTime = 0;

    /**
     * This is a handle to the currently running
     * communications thread. It is protected
     * so that the JavaCGIBridgePool objects can
     * stop timed out threads in its clean up thread.
     */ 
    protected Thread _thread;

    /**
     * This is a byte stream that contains object
     * serialization data in-memory. It is protected
     * so that it can be accessed by the serialization
     * API methods in JavaCGIBridgeExtension class.
     */
    protected ByteArrayOutputStream _serialStream;

    /**
     * This is an object stream that contains
     * a decorator for the byte stream which contains
     * serialized objects for POSTing.  It is protected
     * so that its contents can be accessed by 
     * the serialization API methods in the 
     * JavaCGIBridgeExtension class.
     */
    protected ObjectOutputStream _serialObjectStream;

    // PRIVATE INSTANCE VARIABLES

    /**
     * The field separator.  When a CGI script or HTML file
     * returns data, then the getParsedData() method will
     * know how to separate fields by looking at this variable.
     *
     * The default value is "|" (pipe). This is
     * considered sufficiently unlikely to appear inside of actual
     * field data that it is a pretty good field separator to parse
     * on. If the user of this class needs a different separator, it
     * may be overridden by calling the appropriate method.
     */
    private byte [] _fieldSeparator;

    /**
     * The record separator.  When a CGI script or HTML file
     * returns data, then the getParsedData() method will
     * know how to separate returned rows by looking at this variable.
     *
     * The default value is "\n" (newline). 
     * This is considered sufficiently unlikely to appear inside of
     * actual row data that it is a pretty good row separator to parse
     * on. If the user of this class needs a different separator, it
     * may be overridden by calling the appropriate method.
     */
    private byte [] _recordSeparator;

    /**
     * The top of data separator.  When a CGI script or HTML file
     * returns data, then the getParsedData() method will
     * determine when to start parsing data by encountering this
     * separator.
     *
     * The default value is "<!--start of data-->\n" including a
     * a newline.  That is, a newline is generally expected to follow
     * this separator.
     */
    private byte [] _startDataSeparator;

    /**
     * The bottom of data separator.  When a CGI script or HTML file
     * returns data, then the getParsedData() method will
     * determine when to stop parsing data by encountering this
     * separator.
     *
     * The default value is "<!--end of data-->\n" including
     * a newline.  That is, a newline is generally expected to follow
     * this separator.
     */
    private byte [] _endDataSeparator;

    /**
     * The URL That data is retrieved from.  This is an instance
     * variable rather than a parameter because the data retrieval
     * is done from a launched thread which gets no parameters.
     */
    private URL _threadURL = null;

    /**
     * The HTML form data for the URL that data is retrieved from.  
     * This is an instance variable rather than a parameter because
     * the data retrieval is done from a launched thread which gets
     * no parameters.
     */
    private Hashtable _threadFormVar = null;

    /**
     * This is the returned URL data. It is an instance variable because
     * the run() method of the thread cannot explicitly return data.
     */
    private byte [] _threadRawData = null;

    /** 
     * _parseDataAsVector is a flag letting the HTTP parser know to
     * parse data on the fly as it is coming in.
     */
    private boolean _parseDataAsVector = false;

    /**
     * _parseAsRaw is a flag that indicates whether
     * the Vector of Vectors will contain fields as
     * traditional Java Strings (false, the default) or
     * raw binary byte arrays (true).
     */
    private boolean _parseAsRaw = false;

    /**
     * _threadIdleAtEnd sets the object in the
     * IDLE state automatically when a callOneWay()
     * is used to launch an HTTP session.  This 
     * value is generally set to true by default, but
     * it will be set to false if the user calls the
     * callOneWay() method by telling it that they
     * intend to call fetchNextRecord().  The reason 
     * this is done is that it is possible, in a multi
     * threaded environment that fetchNextRecord() might
     * be called after the whole HTTP session has
     * finished.  In this case, another thread might
     * easily reuse the JavaCGIBridge object in the IDLE
     * state before fetchNextRecord() was called. Therefore,
     * it is imperative that if you intend to use fetchNextRecord()
     * that you tell the callOneWay() function you intend to 
     * do so.
     */
    private boolean _threadIdleAtEnd = true;

    /**
     * _threadOneWay tells the run() method whether
     * it was launched from an asynchronous method 
     * such as callOneWay() or a synchronous method 
     * such as getRawData().  _threadOneWay == true
     * if it was launched from an asynchronous method.
     */
    private boolean _threadOneWay = false; 

    /**
     * This is the number of parsed lines before
     * notifying observers of the latest batch 
     * of parsed Vector records. The default is 0
     * (no notification).
     *
     * @see #getParsedNotifyInterval
     * @see #setParsedNotifyInterval
     */
    private int _parsedNotifyInterval = 0;

    /**
     * This is the number of raw data bytes (usually
     * HTML chars) before notifying observers of the 
     * latest batch of raw data.  The default is
     * 0 (no notification).
     *
     * @see #getRawNotifyInterval
     * @see #setRawNotifyInterval
     */
    private int _rawNotifyInterval = 0;

    /**
     * This integer holds the current processing status
     * of the class. Its value is either SETUP (1),
     * PROCESSING (2), or IDLE (3).
     */
    private int _processingStatus = SETUP;

    /**
     * Constructs the object with new separator values
     */
    public JavaCGIBridge(String field, String record, String startData, String endData) {
        _initJavaCGIBridge(field,record,startData,endData);
    }

    /**
     * Constructs the object with no initialization. This is the default
     * (empty) constructor.
     */
    public JavaCGIBridge() {
        _initJavaCGIBridge("|","\n","<!--start of data-->\n","<!--end of data-->\n");
    } // Empty constructor

    /**
     * Adds a form variable, value pair to the passed Hashtable.
     *
     * @param ht the Hashtable that contains the form variable/value pairs
     * @param formKey the String that contains the form variable to add
     * @param formValue the String that contains the form value to add
     */
    public void addFormValue(Hashtable ht, String formKey, String formValue) {
        Vector vValues = null;

        if (formValue != null && formValue.length() > 0) {
            if (ht.containsKey(formKey)) {
                vValues = (Vector)ht.get(formKey);
            } else {
                vValues = new Vector();
            }
            vValues.addElement(formValue);
            ht.put(formKey, vValues);
        }
    } // End of addFormValue

    /** 
     * Takes the parsed data returned from the getParsedData method
     * and changes it to a Hashtable of key, value pairs where the first
     * Vector entry of each Vector record is the key and the rest of the
     * second Vector entry for each record is the value of the Hashtable.
     *
     * @param vectorOfVectors the Vector of Vectors for the parsed data
     * @return Hashtable containing converted variable/value pairs
     * @see #getParsedData
     */ 
    public Hashtable getKeyValuePairs(Vector vectorOfVectors) {
        Hashtable h = new Hashtable();
        Vector v = null;

        for (Enumeration e = vectorOfVectors.elements(); e.hasMoreElements();) {
            v = (Vector)e.nextElement();
            h.put((String)v.elementAt(0), (String)v.elementAt(1));
        }
        return h;
    } // End of getKeyValuePairs

    /**
     * Returns parsed data in the form of a Vector of Vectors containing
     * the returned fields inside of a Vector of returned rows.
     *
     * @param u URL to get parsed data from.
     * @return Vector (records) of vectors (fields) of parsed data
     * @exception JavaCGIBridgeTimeOutException If the retrieval times out
     * @see #getRawData
     * @see #callOneWay
     * @see #getParseAsRaw
     * @see #setParseAsRaw
     * @see #getParsedNotifyInterval
     * @see #setParsedNotifyInterval
     */
    public Vector getParsedData(URL u) throws JavaCGIBridgeTimeOutException {
        return (getParsedData(u, null));
    }

    /**
     * Returns parsed data in the form of a Vector of Vectors containing
     * the returned fields inside of a Vector of returned rows. This form
     * POSTs the HTML Form variable data to the URL.
     *
     * @param u URL to get parsed data from.
     * @param ht Hashtable contains form variables to POST
     * @return Vector (records) of vectors (fields) of parsed data
     * @exception JavaCGIBridgeTimeOutException If the retrieval times out
     * @see #getRawData
     * @see #callOneWay
     * @see #getParseAsRaw
     * @see #setParseAsRaw
     * @see #getParsedNotifyInterval
     * @see #setParsedNotifyInterval
     */
    public Vector getParsedData(URL u, Hashtable ht) 
      throws JavaCGIBridgeTimeOutException {

        // getParsedData calls getRawData to obtain
        // the actual HTML text

        // Notice that we turn on the flag to parse
        // vector data inside the getRawData block
        // 
        // We need to do this in order to provide a 
        // mechanism for returning partial results to
        // an observer.
        //
        _parseDataAsVector = true;
        _getData(u,ht);
        _parseDataAsVector = false;

        // NOTE: We have to save the vector parsed data
        // because as soon as we set the object to the IDLE
        // state, another thread could start using the JavaCGIBridge
        Vector v = _vectorParsedData;
        setStatus(IDLE);
        return v;
    }

    /**
     * Returns raw HTML data as a String from the passed URL.
     *
     * @param u URL to get raw HTML from.
     * @return String containing plain HTML text
     * @exception JavaCGIBridgeTimeOutException If the retrieval times out
     * @see #getParsedData
     * @see #callOneWay
     * @see #getRawNotifyInterval
     * @see #setRawNotifyInterval
     */
    public byte [] getRawData(URL u) throws JavaCGIBridgeTimeOutException {
        return (getRawData(u,null));
    }

    /**
     * Returns raw HTML data as a String from the passed URL and list
     * of Form variable/value pairs stored in a Hashtable.  This form
     * POSTs the HTML Form variable data to the URL.
     *
     * @param u URL to get raw HTML from.
     * @param ht Hashtable contains form variables to POST
     * @return String containing plain HTML text
     * @exception JavaCGIBridgeTimeOutException If the retrieval times out
     * @see #getParsedData
     * @see #callOneWay
     * @see #getRawNotifyInterval
     * @see #setRawNotifyInterval
     */
    public byte [] getRawData(URL u, Hashtable ht)
      throws JavaCGIBridgeTimeOutException 
    {

        _getData(u,ht);
        // NOTE: We have to save the vector parsed data
        // because as soon as we set the object to the IDLE
        // state, another thread could start using the JavaCGIBridge
        byte [] b = _threadRawData;
        setStatus(IDLE);
        return b;
    } // end of getRawData

    /**
     * Calls the URL but returns immediately.  This is
     * called a ONEWAY operation.  This method is used
     * in one of three circumstances.
     * <P>
     * [1] You want to "fire and forget" an operation such
     * as an update to a file or database on the web server
     * without worrying about a return code or whether the
     * operation even succedded.
     * <P>
     * [2] You want to set up an observer interface to
     * the bridge to notify yourself as data comes in
     * instead of blocking the whole program while waiting
     * for the data.
     * <P>
     * [3] You want to call fetchNextRecord method in
     * the JavaCGIBridgeExtension class.  Basically, the 
     * callOneWay() operation returns immediately so that
     * it leaves your program free to loop through the
     * fetchNextRecord() method. Note that if you use
     * callOneWay for this purpose, it is recommended that
     * you call the version where you set "fetchData" equal
     * to true so that the Object is not set to the IDLE
     * state prematurely.  This is discussed in further
     * detail in the documentation for fetchNextRecord()
     * in the JavaCGIBridgeExtension class.
     *
     * @param u URL to get raw HTML from.
     * @see #getRawData
     * @see #getParsedData
     * @see #getRawNotifyInterval
     * @see #setRawNotifyInterval
     * @see #getParsedNotifyInterval
     * @see #setParsedNotifyInterval 
     */
    public void callOneWay(URL u) {
        callOneWay(u,null);
    } 

    /**
     * Calls the URL but returns immediately.  This is
     * called a ONEWAY operation.  This method is used
     * in one of three circumstances.
     * <P>
     * [1] You want to "fire and forget" an operation such
     * as an update to a file or database on the web server
     * without worrying about a return code or whether the
     * operation even succedded.
     * <P>
     * [2] You want to set up an observer interface to
     * the bridge to notify yourself as data comes in
     * instead of blocking the whole program while waiting
     * for the data.
     * <P>
     * [3] You want to call fetchNextRecord method in
     * the JavaCGIBridgeExtension class.  Basically, the
     * callOneWay() operation returns immediately so that
     * it leaves your program free to loop through the
     * fetchNextRecord() method. Note that if you use
     * callOneWay for this purpose, it is recommended that
     * you call the version where you set "fetchData" equal
     * to true so that the Object is not set to the IDLE
     * state prematurely.  This is discussed in further
     * detail in the documentation for fetchNextRecord()
     * in the JavaCGIBridgeExtension class.
     *
     * @param u URL to get raw HTML from.
     * @param fetchData Set to true if you intend to
     *                  use the fetchNextRecord() method.
     *                  This avoids the thread automatically
     *                  setting the state of the object to
     *                  IDLE when the thread completes.  This
     *                  is necessary because it is possible that
     *                  fetchNextRecord has not completed
     *                  fetching all the records by the time
     *                  the communications thread ends.
     * @see #getRawData
     * @see #getParsedData
     * @see #getRawNotifyInterval
     * @see #setRawNotifyInterval
     * @see #getParsedNotifyInterval
     * @see #setParsedNotifyInterval
     */
    public void callOneWay(URL u, boolean fetchData) {
        callOneWay(u,null,fetchData);
    }

    /**
     * Calls the URL but returns immediately.  This is
     * called a ONEWAY operation.  This method is used
     * in one of three circumstances.
     * <P>
     * [1] You want to "fire and forget" an operation such
     * as an update to a file or database on the web server
     * without worrying about a return code or whether the
     * operation even succedded.
     * <P>
     * [2] You want to set up an observer interface to
     * the bridge to notify yourself as data comes in
     * instead of blocking the whole program while waiting
     * for the data.
     * <P>
     * [3] You want to call fetchNextRecord method in
     * the JavaCGIBridgeExtension class.  Basically, the
     * callOneWay() operation returns immediately so that
     * it leaves your program free to loop through the
     * fetchNextRecord() method. Note that if you use
     * callOneWay for this purpose, it is recommended that
     * you call the version where you set "fetchData" equal
     * to true so that the Object is not set to the IDLE
     * state prematurely.  This is discussed in further
     * detail in the documentation for fetchNextRecord()
     * in the JavaCGIBridgeExtension class.
     *
     * @param u URL to get raw HTML from.
     * @param ht Hashtable contains form variables to POST
     * @see #getRawData
     * @see #getParsedData
     * @see #getRawNotifyInterval
     * @see #setRawNotifyInterval
     * @see #getParsedNotifyInterval
     * @see #setParsedNotifyInterval
     */
    public void callOneWay(URL u, Hashtable ht) {
        callOneWay(u,ht,false);
    }

    /**
     * Calls the URL but returns immediately.  This is
     * called a ONEWAY operation.  This method is used
     * in one of three circumstances.
     * <P>
     * [1] You want to "fire and forget" an operation such
     * as an update to a file or database on the web server
     * without worrying about a return code or whether the
     * operation even succedded.
     * <P>
     * [2] You want to set up an observer interface to
     * the bridge to notify yourself as data comes in
     * instead of blocking the whole program while waiting
     * for the data.
     * <P>
     * [3] You want to call fetchNextRecord method in
     * the JavaCGIBridgeExtension class.  Basically, the
     * callOneWay() operation returns immediately so that
     * it leaves your program free to loop through the
     * fetchNextRecord() method. Note that if you use
     * callOneWay for this purpose, it is recommended that
     * you call the version where you set "fetchData" equal
     * to true so that the Object is not set to the IDLE
     * state prematurely.  This is discussed in further
     * detail in the documentation for fetchNextRecord()
     * in the JavaCGIBridgeExtension class.
     *
     * @param u URL to get raw HTML from.
     * @param ht Hashtable contains form variables to POST
     * @param fetchData Set to true if you intend to
     *                  use the fetchNextRecord() method.
     *                  This avoids the thread automatically
     *                  setting the state of the object to
     *                  IDLE when the thread completes.  This
     *                  is necessary because it is possible that
     *                  fetchNextRecord has not completed
     *                  fetching all the records by the time
     *                  the communications thread ends.
     * @see #getRawData
     * @see #getParsedData
     * @see #getRawNotifyInterval
     * @see #setRawNotifyInterval
     * @see #getParsedNotifyInterval
     * @see #setParsedNotifyInterval
     */
    public void callOneWay(URL u, Hashtable ht, boolean fetchData) {
        _threadURL = u;
        _threadFormVar = ht;
        _threadBaseTime = System.currentTimeMillis();

        // We want to make sure that
        // if we are setting up for fetching
        // records that the data is parsed
        // and the JavaCGIBridge is not set
        // to IDLE until the last record is fetched.
        _parseDataAsVector = fetchData || (_parsedNotifyInterval != 0);
        _threadIdleAtEnd = !fetchData;
        _threadOneWay = true;

        _thread= new Thread(this);
        _threadCompleted = false;
        _thread.start();
    }

    /**
     * Returns the actual communication time out in milliseconds
     * for the object. The default is 10 seconds.
     *
     * When the object retrieves data from a URL, it must get
     * the data within timeout milliseconds or a
     * JavaCGIBridgeTimeOutException is thrown.
     * 
     * @return communication time out in milliseconds
     * @see #setTimeOut
     */
    public int getTimeOut() { 
        return _threadTimeOut; 
    }

    /**
     * Sets the actual communication time out in milliseconds
     * for the object.  The default is 10 seconds.
     *
     * When the object retrieves data from a URL, it must get
     * the data within timeout milliseconds or a
     * JavaCGIBridgeTimeOutException is thrown.
     *
     * @param t communication time out in milliseconds
     * @see #getTimeOut
     */
    public void setTimeOut(int t) { 
        _threadTimeOut = t; 
    }

    /**
     * Sets the field separator for the object.  When getParsedData
     * method is called, the object uses the field separator to determine 
     * where fields in a returned record of the raw HTML result set
     * begin and end.
     *
     * @param s String containing new delimiting separator
     * @see #getParsedData
     * @see #getFieldSeparator
     */
    public void setFieldSeparator(String s) { _fieldSeparator = _getBytes(s); }

    /**
     * Returns the field separator for the object.  When getParsedData
     * method is called, the object uses the field separator to determine 
     * where fields in a returned record of the raw HTML result set
     * begin and end.
     *
     * @return Separator string
     * @see #getParsedData
     * @see #setFieldSeparator
     */
    public String getFieldSeparator() { return new String(_fieldSeparator, 0); }

    /**
     * Sets the record separator for the object.  When getParsedData
     * method is called, the object uses the record separator to determine 
     * where records/rows of the raw HTML result set
     * begin and end.
     *
     * @param s String containing new delimiting separator
     * @see #getParsedData
     * @see #getRecordSeparator
     */
    public void setRecordSeparator(String s) { _recordSeparator = _getBytes(s); }

    /**
     * Returns the record separator for the object.  When getParsedData
     * method is called, the object uses the record separator to determine 
     * where records/rows of the raw HTML result set
     * begin and end.
     *
     * @return Separator string
     * @see #getParsedData
     * @see #setRecordSeparator
     */
    public String getRecordSeparator() { return new String(_recordSeparator, 0); }

    /**
     * Sets the start of data separator for the object.  When getParsedData
     * method is called, the object uses the start of data separator to determine 
     * where the rows of data inside the raw HTML output actually begin.
     *
     * @param s String containing new delimiting separator
     * @see #getParsedData
     * @see #getStartDataSeparator
     */
    public void setStartDataSeparator(String s) { _startDataSeparator = _getBytes(s); }

    /**
     * Returns the start of data separator for the object.  When getParsedData
     * method is called, the object uses the start of data separator to determine
     * where the rows of data inside the raw HTML output actually begin.
     *
     * @return Separator string
     * @see #getParsedData
     * @see #setStartDataSeparator
     */
    public String getStartDataSeparator() { return new String(_startDataSeparator, 0); }

    /**
     * Sets the end of data separator for the object.  When getParsedData
     * method is called, the object uses the end of data separator to determine
     * where the rows of data inside the raw HTML output actually end.
     *
     * @param s String containing new delimiting separator
     * @see #getParsedData
     * @see #getEndDataSeparator
     */
    public void setEndDataSeparator(String s) { _endDataSeparator = _getBytes(s); }

    /**
     * Returns the end of data separator for the object.  When getParsedData
     * method is called, the object uses the end of data separator to determine
     * where the rows of data inside the raw HTML output actually end.   
     *
     * @return Separator string
     * @see #getParsedData
     * @see #setEndDataSeparator
     */
    public String getEndDataSeparator() { return new String(_endDataSeparator, 0); }

    /**
     * Returns an integer indicating the number of records which will
     * trigger a notification to observers with the contents of what
     * has been parsed so far.  When this is set to a positive number,
     * it will also notify when the records have finished being parsed
     * at the end of the HTTP response.  When this is set to a negative
     * number, no periodic notifications will be sent, however one will 
     * be sent at the end of the HTTP response.
     *
     * @return int Number of records between parse notifications. Set to 0
     *             to turn off parsing notification, 
     *             Set to a negative number to
     *             only send notifications at the end.
     * @see #setParsedNotifyInterval
     */
    public int getParsedNotifyInterval() {
      return _parsedNotifyInterval;
    }

    /**
     * Sets an integer indicating the number of records which will
     * trigger a notification to observers with the contents of what
     * has been parsed so far.  When this is set to a positive number,
     * it will also notify when the records have finished being parsed
     * at the end of the HTTP response.  When this is set to a negative
     * number, no periodic notifications will be sent, however one will 
     * be sent at the end of the HTTP response.
     *
     * @param n Number of records between parse notifications. Set to 0
     *             to turn off parsing notification,
     *             Set to a negative number to
     *             only send notifications at the end.
     * @see #getParsedNotifyInterval
     */
    public void setParsedNotifyInterval(int n) {
      _parsedNotifyInterval = n;
    }

    /**
     * Returns an integer indicating the number of raw content bytes
     * from the HTTP response which will
     * trigger a notification to observers with the contents of what
     * has been read so far.  When this is set to a positive number,
     * it will also notify when the HTTP response has finished.
     * When this is set to a negative
     * number, no periodic notifications will be sent, however one will 
     * be sent at the end of the HTTP response.
     *
     * @return int Number of bytes between notifications. Set to 0
     *             to turn off notification after lines of HTML
     *             sent in the HTTP response, 
     *             Set to a negative number to
     *             only send notifications at the end.
     * @see #setRawNotifyInterval
     */
    public int getRawNotifyInterval() {
      return _rawNotifyInterval;
    }

    /**
     * Sets an integer indicating the number of raw content bytes
     * from the HTTP response which will
     * trigger a notification to observers with the contents of what
     * has been read so far.  When this is set to a positive number,
     * it will also notify when the HTTP response has finished.
     * When this is set to a negative
     * number, no periodic notifications will be sent, however one will 
     * be sent at the end of the HTTP response.
     *
     * @param n Number of bytes between notifications. Set to 0
     *             to turn off notification after bytes of HTML
     *             sent in the HTTP response, 
     *             Set to a negative number to
     *             only send notifications at the end.
     * @see #getRawNotifyInterval
     */
    public void setRawNotifyInterval(int n) {
      _rawNotifyInterval = n;
    }

    /**
     * Returns true or false depending on
     * whether the JavaCGIBridge will parse the fields
     * in the HTTP response as Strings (default) or 
     * as raw data byte arrays.  Use byte arrays if you
    public void setRawNotifyInterval(int n) {
      _rawNotifyInterval = n;
    }

    /**
     * Returns true or false depending on
     * whether the JavaCGIBridge will parse the fields
     * in the HTTP response as Strings (default) or 
     * as raw data byte arrays.  Use byte arrays if you
     * will be transferring binary data such as a jpeg
     * or gif image.
     *
     * @return boolean Flag indicating parsing as
     *                 raw byte array or as Strings
     * @see #setParseAsRaw
     */
    public boolean getParseAsRaw() {
      return _parseAsRaw;
    }

    /** 
     * Sets the flag that indicates whether the 
     * JavaCGIBridge will parse the HTTP response
     * data as raw byte array or as String (default).
     *
     * @param b Flag indicating true or false
     * @see #getParseAsRaw
     */
    public void setParseAsRaw(boolean b) {
      _parseAsRaw = b;
    }

    /**
     * Returns the processing status of the JavaCGIBridge
     * objects.  It will be equal to SETUP (1), PROCESSING (2),
     * or IDLE (3) public static class variables.  
     *
     * @return int Status code
     * @see #setStatus
     */ 
    public int getStatus() {
        return _processingStatus;
    } // end of getStatus

    /** 
     * Sets the processing status of the JavaCGIBridge
     * object.  You will set it to SETUP (1), PROCESSING (2),
     * or IDLE (3) public static class variables.
     *
     * @param status Status code
     * @see #getStatus
     */
    public void setStatus(int status) {
        if (status < 1 || status > 3) return;
        _processingStatus = status;
    } // end of setStatus

    /**
     * This run thread asynchronously POSTs and GETs data from
     * a URL and places the contents into the _threadRawData variable.
     *
     * Since Threads do not return or pass parameter values, instance
     * variables in this object are used to maintain state.  This method
     * is only public so that the Thread class can launch the thread.
     *
     * @see #getRawData
     */
    public void run() {
        try {
        _threadRawData = _getHttpRequestInThread(
                        _threadURL, _threadFormVar);
        } catch (JavaCGIBridgeTimeOutException e) {
            e.printStackTrace();
            System.out.println("JavaCGIBridgeTimeOutException thrown!");
        } finally {
            _threadCompleted = true; // set completed flag
            synchronized(this) { notifyAll(); } 
              // ends the wait() for thread to end in getRawData
            if (_threadOneWay && _threadIdleAtEnd) {
                setStatus(IDLE);
                _threadOneWay = false;
                _threadIdleAtEnd = false;
                _parseDataAsVector = false;
            }
        }
    } // end of run -- The thread runs

    /** 
     * Returns the form variables inside the hashtable as a 
     * URL encoded string of parameters for a CGI based program
     * to process.
     *
     * @param ht Hashtable containing form variables
     * @return URLencoded string of HTML form variables & values
     * @see #addFormValue
     */
    public String getURLEncodedHashtable(Hashtable ht) {
        StringBuffer encodedString = null;
        Vector vFormValues = null;

        // First, we enumerate through the keys (Form variables)
        for (Enumeration eKeys = ht.keys() ; eKeys.hasMoreElements() ;) {
            String formVariable = (String)eKeys.nextElement();

            vFormValues = (Vector)ht.get(formVariable);
            // Now, we have to enumerate through the values for the form variables.
            //
            // NOTE: It IS entirely possible that a form variable has MANY values
            // to simulate events such as multiple checkboxes or multiple select <SELECT>
            // form elements in an HTML form.
            for (Enumeration eVals = vFormValues.elements() ; eVals.hasMoreElements() ;) {
                String formValue = (String)eVals.nextElement();

                if (encodedString != null) {
                    encodedString.append("&");
                } else {
                    encodedString = new StringBuffer();
                }

                encodedString.append(URLEncoder.encode(formVariable) + "=" +
                    URLEncoder.encode(formValue));
            }
        }
        return encodedString.toString();
    } // End of getURLEncodedHashtable

    /**
     * Returns the HTTP Request data to the thread that
     * was launched to GET/POST data for a URL.  This is
     * a private method.
     *
     * @param u URL to retrieve and post data for
     * @param ht Form variables to send to URL
     * @return String containing retrieved HTML text
     * @exception JavaCGIBridgeTimeOutException If the retrieval times out
     * @see #run
     */
    private byte [] _getHttpRequestInThread(URL u, Hashtable ht) throws JavaCGIBridgeTimeOutException {
        String postContent = null;
        ByteArrayOutputStream rawData = new ByteArrayOutputStream();

        if (ht != null) {
            postContent = getURLEncodedHashtable(ht);
        }

        try {
            URLConnection       urlConn;
            DataOutputStream    dos;
            InputStream         is;

            // Establish the URL connection
            urlConn = u.openConnection();

            // We always want some input from the CGI script
            // in general even if it is just a success story
            urlConn.setDoInput (true);

            // If ht was not null, then we know there is
            // data to be posted.
            boolean serialOn = false;
            if (_serialStream != null && _serialStream.size() > 0) {
                serialOn = true;
            }
            if (ht != null || serialOn) {
                urlConn.setDoOutput(true);
            }

            // Turn off caching as this may screw with
            // our dynamic programs

            urlConn.setUseCaches (false);
            // Specify the content type
            urlConn.setRequestProperty
               ("Content-type", "application/x-www-form-urlencoded");

            // Send the POST data if we are writing
            if (ht != null || serialOn) {
                dos = new DataOutputStream(urlConn.getOutputStream());
                if (ht != null) {
                    dos.writeBytes(postContent);
                } else {
                    dos.write(_serialStream.toByteArray(),0,_serialStream.toByteArray().length);
                }
                dos.flush();
                dos.close();
                _serialStream = null;
                _serialObjectStream = null;
            }

            is = urlConn.getInputStream();

            if (_parseDataAsVector == true) {
                _vectorParsedData = new Vector();
            }
            int startDataIndex = -1, endDataIndex = -1; 
            int lastRecordIndex = 0;
            int lastFoundIndex = 0;
            boolean stopParsingRecords = false;

            int c;
            boolean stillReading = true;
            int maxSize = 100;
            int bufSize = 0;
            _fetchCount = 0;
            int vectorLineCount = 0;
            int htmlByteCount = 0;

            while (stillReading)
            {
                c = is.read();
                if (c == -1) {
                  stillReading = false;
                } else {
                  if (bufSize >= maxSize) bufSize = 0;
                  bufSize++;
                  rawData.write(c);
                }

                if (_parseDataAsVector == true && (bufSize == maxSize || !stillReading)) {
                    if (startDataIndex == -1) {
                        startDataIndex = _getIndex(rawData,_startDataSeparator,0,-1);
                        if (startDataIndex != -1) 
                            lastFoundIndex = startDataIndex + _startDataSeparator.length;
                    }

                    int recordIndex = 0;
                    while (startDataIndex != -1 && !stopParsingRecords && recordIndex != -1) { // We can begin parsing!


                    if (endDataIndex == -1 && startDataIndex != -1)
                        endDataIndex = _getIndex(rawData,_endDataSeparator,lastFoundIndex,-1);

                        recordIndex = _getIndex(rawData,_recordSeparator,lastFoundIndex,-1);

                        if (recordIndex > endDataIndex && endDataIndex != -1)
                            break;

                        int startIndex = lastFoundIndex;
                        // Take care of case where _recordSeparator was forgotten
                        // before _endDataSeparator.
                        if ((recordIndex == -1 && endDataIndex != -1 && 
                            (endDataIndex - lastRecordIndex) != _recordSeparator.length) ||
                            (recordIndex != -1 && recordIndex > endDataIndex && endDataIndex != -1)) {
                            recordIndex = endDataIndex;
                            lastFoundIndex = recordIndex + _endDataSeparator.length;
                            stopParsingRecords = true;
                        } else if (recordIndex != -1) {
                            lastFoundIndex = recordIndex + _recordSeparator.length;
                        }
                        if (recordIndex != -1) {
                            lastRecordIndex = recordIndex;
                            _vectorParsedData.addElement(new Vector());
                            int fieldIndex = startIndex;
                            while (fieldIndex != -1) {
                                int beginFieldIndex = fieldIndex;
                                fieldIndex = _getIndex(rawData,_fieldSeparator,fieldIndex,recordIndex);
                                Object newElement;
                                byte [] b = null;
                                if (fieldIndex != -1) {
                                    b = new byte[fieldIndex - beginFieldIndex];
                                    System.arraycopy(rawData.toByteArray(),beginFieldIndex,b,0,
                                                       fieldIndex - beginFieldIndex);
                                    fieldIndex = fieldIndex + _fieldSeparator.length;
                                } else {
                                    b = new byte[recordIndex - beginFieldIndex];
                                    System.arraycopy(rawData.toByteArray(),beginFieldIndex,b,0,
                                                       recordIndex - beginFieldIndex);
                                }
                                if (_parseAsRaw) {
                                  newElement = b;
                                } else {
                                  newElement = new String(b, 0);
                                }
                                ((Vector)_vectorParsedData.lastElement()).addElement(newElement);
                            } // Processing fields in a record
                            vectorLineCount++;
                            if (_parsedNotifyInterval > 0 && 
                                vectorLineCount % _parsedNotifyInterval == 0)
                                    _notifyObservers(_vectorParsedData, false);
                        } // Processing recordIndex

                    } // end of while we are still parsing records

                    if (endDataIndex == -1 && startDataIndex != -1) 
                        endDataIndex = _getIndex(rawData,_endDataSeparator,lastFoundIndex,-1);

                    if (endDataIndex != -1 && 
                        lastRecordIndex + _recordSeparator.length >= endDataIndex) 
                          stopParsingRecords = true;

                }

                htmlByteCount++;
                if (_rawNotifyInterval > 0 &&
                    htmlByteCount % _rawNotifyInterval == 0) 
                        _notifyObservers(rawData.toByteArray(), false);

            } // end of while reading lines from HTTP connection

            is.close ();

            if (_rawNotifyInterval != 0)
                _notifyObservers(rawData.toByteArray(), true);

            if (_parsedNotifyInterval != 0)
                _notifyObservers(_vectorParsedData, true);

            
        } catch (MalformedURLException me) {
            System.err.println("MalformedURLException: " + me);
        } catch (IOException ioe) {
            System.err.println("IOException: " + ioe.getMessage());
            System.err.println("Could be the result of a bad URL: " + u);
        }

            return rawData.toByteArray();

    } // end of _getHttpRequestInThread

    /**
     * Retrieves the index in the ByteArrayOutputStream
     * of where a string to search for is located. The current
     * algorithm is based loosely off of the code in 
     * String.indexOf in the java.lang.String source
     * code.
     *
     */
    private int _getIndex(ByteArrayOutputStream baos, byte [] toLookFor, int begin, int end) {
    // Code here is based loosely on String.indexOf code in java.lang.String source
        byte [] buf = baos.toByteArray();

        if (end == -1) end = buf.length;
        if (begin >= end) return -1;

        byte v1[] = new byte[end - begin];
        //arraycopy(src,src_posdest,dest_pos,length);
        System.arraycopy(buf,begin,v1,0,end - begin);
        byte v2[] = toLookFor; // toLookFor.getBytes();

        int max = v1.length - v2.length;

      test:
        for (int i = 0; i <= max ; i++) {
            int n = v2.length;
            int j = i;
            int k = 0;
            while (n-- != 0) {
                if (v1[j++] != v2[k++]) {
                    continue test;
                }
            }
            // Must return the correct offset...
            return begin + i;
       }
       return -1;
    } // end of _getIndex

    /**
     * Notifies the observers of the data that is being
     * parsed or received in the communications thread.
     * It constructs a JavaCGIBridgeNotify object to
     * send the information including whether or not
     * the JavaCGIBridge timed out or whether the end of
     * data has been reached.
     *
     * @param o Object containing data to send out such 
     *          as the latest Vector or the latest byte array
     *          of data.
     * @param endOfData indicates whether the end of data was reached
     * @exception JavaCGIBridgeTimeOutException If the retrieval times out
     */
    private void _notifyObservers(Object o, boolean endOfData) throws JavaCGIBridgeTimeOutException{
        this.setChanged();
        if (_threadTimeOut < (System.currentTimeMillis() - _threadBaseTime)) {
            notifyObservers(new JavaCGIBridgeNotify(new JavaCGIBridgeTimeOutException(), endOfData));
            _threadOneWay = false;
            _parseDataAsVector = false;
            _threadIdleAtEnd = false;
            throw new JavaCGIBridgeTimeOutException();
        } else {
            notifyObservers(new JavaCGIBridgeNotify(o, endOfData));
        } 
        _threadBaseTime = System.currentTimeMillis();
    }

    /**
     * Initializes the object with new separator values.
     * It is called from the constructors of this object.
     *
     * @param field Field separator
     * @param record Record separator
     * @param startData Start of Data separator
     * @param endData End of Data separator
     */
    private void _initJavaCGIBridge(String field, String record, String startData, String endData) {
        _fieldSeparator = _getBytes(field);
        _recordSeparator = _getBytes(record);
        _startDataSeparator = _getBytes(startData);
        _endDataSeparator = _getBytes(endData);
    }

    /**
     * Starts up the communications thread
     * and gets the data from it in a synchronous
     * manner. Thus, this method blocks until
     * it gets all the data.
     *
     * It also implements a timeout mechanism
     * using wait() and notify().  It is basically
     * the same as Thread.join() except that it
     * fixes the broken Thread.join() implementation
     * in Netscape browsers.
     *
     * @param u URL to get parsed data from.
     * @param ht Hashtable contains form variables to POST
     * @exception JavaCGIBridgeTimeOutException If the retrieval times out
     * @see #getParsedData
     * @see #getRawData
     */
    private void _getData(URL u, Hashtable ht)
      throws JavaCGIBridgeTimeOutException
    {

        // This is just a quick check, it shouldn't normally happen
        // though... Generally only a OneWay will cause problem.
        //
        synchronized (this) {
            if (_processingStatus == PROCESSING) {
                throw new JavaCGIBridgeStillProcessingException();
            } else {
                _processingStatus = PROCESSING;
            }
        }

        // We set up the information for passing to the thread
        // ahead of time.
        _threadURL = u;
        _threadFormVar = ht;

        // Create a new thread and flag the fact that the thread
        // did not get us any data yet. Then start the thread.
        _thread = new Thread(this);
        _threadCompleted = false;
        _thread.start();

        // We calculate the current time that the thread started.
        // The delay to wait will be calculated below.
        _threadBaseTime = System.currentTimeMillis();
        long delay = 0;
        synchronized(this) {
            try {
                while(_thread.isAlive() && _threadCompleted == false) {
                    delay = _threadTimeOut -
                        (System.currentTimeMillis() - _threadBaseTime);
                    if (delay <= 0) break;
                    wait(delay);
                } // End of while
            } catch (InterruptedException e) { /* nothing */ }
        } // end of synchronized block

        if (_threadCompleted == false) {
            _thread.stop(); // No effect if already done but stops if timed out
            throw new JavaCGIBridgeTimeOutException();
        }

    } // end of _getData

    /**
     * This private routine converts strings
     * to byte arrays in a way that is more
     * convenient than the traditional JDK 1.0.2
     * method. In otherwords, this is a wrapper
     * for the more complex version of the
     * String to byte array converter method.
     *
     * @param s String to convert to bytes
     * @return byte[] converted byte array
     */
    private byte [] _getBytes(String s) {
        byte [] b = new byte[s.length()];
        s.getBytes(0, s.length(), b, 0);
        return b;
    } // end of _getBytes
} // End of JavaCGIBridge class