Re: Re: [INTERFACES] Re: [PATCHES] Re: Fixes and enhancements to JDBC driver(take 2)

Here is a context diff of the current CVS and the user-supplied version
for review.

> On Fri, 2 Feb 2001, Peter T Mount wrote:
>
> > > What is the final resolution of this for 7.1?  I noticed that you have
> > > essentially commented out the byte array pooling code in current
> > > sources.  Given that 7.1 is rapidly coming to a close, is this
> > > something
> > > that is expected to get fixed and reenabled for 7.1, or is this going
> > > to wait for 7.2.
> >
> > I commented it out to make sure that when 7.1 was released the driver
> > would at least work (with it enabled we would have one broken driver).
>
> At Microstate, we fixed the byte pooling code when it was still in
> PG_Stream. We patched this and submitted our patch to pgsql-patches, and
> it was accepted.  It did have some nasty off-by-one errors. It does not
> look like the fixes we did made it into the new, separate
> org.postgresql.core.BytePoolDim[12] classes.
>
> We've been using this code on an application with many tens of thousands
> of queries per day for more than a month without any problems.
>
> I've attached the production PG_Stream.java class that we've been using.
>
>  --
>  Richard Bullington-McGuire  <rbulling@microstate.com>
>  Chief Technology Officer, The Microstate Corporation
>  Phone: 703-796-6446  URL: http://www.microstate.com/
>  PGP key IDs:    RSA: 0x93862305   DH/DSS: 0xDAC3028E
>
Content-Description:

[ Attachment, skipping... ]


--
  Bruce Momjian                        |  http://candle.pha.pa.us
  pgman@candle.pha.pa.us               |  (610) 853-3000
  +  If your life is a hard drive,     |  830 Blythe Avenue
  +  Christ can be your backup.        |  Drexel Hill, Pennsylvania 19026
*** /bjm/0    Fri Feb  2 12:51:43 2001
--- PG_Stream.java    Thu Jan 18 13:16:00 2001
***************
*** 6,11 ****
--- 6,12 ----
  import java.util.*;
  import java.sql.*;
  import org.postgresql.*;
+ import org.postgresql.core.*;
  import org.postgresql.util.*;

  /**
***************
*** 22,31 ****
    private Socket connection;
    private InputStream pg_input;
    private BufferedOutputStream pg_output;
!
      BytePoolDim1 bytePoolDim1 = new BytePoolDim1();
      BytePoolDim2 bytePoolDim2 = new BytePoolDim2();
!
    /**
     * Constructor:  Connect to the PostgreSQL back end and return
     * a stream connection.
--- 23,32 ----
    private Socket connection;
    private InputStream pg_input;
    private BufferedOutputStream pg_output;
!
      BytePoolDim1 bytePoolDim1 = new BytePoolDim1();
      BytePoolDim2 bytePoolDim2 = new BytePoolDim2();
!
    /**
     * Constructor:  Connect to the PostgreSQL back end and return
     * a stream connection.
***************
*** 37,52 ****
    public PG_Stream(String host, int port) throws IOException
    {
      connection = new Socket(host, port);
!
      // Submitted by Jason Venner <jason@idiom.com> adds a 10x speed
      // improvement on FreeBSD machines (caused by a bug in their TCP Stack)
      connection.setTcpNoDelay(true);
!
      // Buffer sizes submitted by Sverre H Huseby <sverrehu@online.no>
      pg_input = new BufferedInputStream(connection.getInputStream(), 8192);
      pg_output = new BufferedOutputStream(connection.getOutputStream(), 8192);
    }
!
    /**
     * Sends a single character to the back end
     *
--- 38,53 ----
    public PG_Stream(String host, int port) throws IOException
    {
      connection = new Socket(host, port);
!
      // Submitted by Jason Venner <jason@idiom.com> adds a 10x speed
      // improvement on FreeBSD machines (caused by a bug in their TCP Stack)
      connection.setTcpNoDelay(true);
!
      // Buffer sizes submitted by Sverre H Huseby <sverrehu@online.no>
      pg_input = new BufferedInputStream(connection.getInputStream(), 8192);
      pg_output = new BufferedOutputStream(connection.getOutputStream(), 8192);
    }
!
    /**
     * Sends a single character to the back end
     *
***************
*** 59,69 ****
        //byte b[] = new byte[1];
        //b[0] = (byte)val;
        //pg_output.write(b);
!
        // Optimised version by Sverre H. Huseby Aug 22 1999 Applied Sep 13 1999
        pg_output.write((byte)val);
    }
!
    /**
     * Sends an integer to the back end
     *
--- 60,70 ----
        //byte b[] = new byte[1];
        //b[0] = (byte)val;
        //pg_output.write(b);
!
        // Optimised version by Sverre H. Huseby Aug 22 1999 Applied Sep 13 1999
        pg_output.write((byte)val);
    }
!
    /**
     * Sends an integer to the back end
     *
***************
*** 74,80 ****
    public void SendInteger(int val, int siz) throws IOException
    {
      byte[] buf = bytePoolDim1.allocByte(siz);
!
      while (siz-- > 0)
        {
      buf[siz] = (byte)(val & 0xff);
--- 75,81 ----
    public void SendInteger(int val, int siz) throws IOException
    {
      byte[] buf = bytePoolDim1.allocByte(siz);
!
      while (siz-- > 0)
        {
      buf[siz] = (byte)(val & 0xff);
***************
*** 82,88 ****
        }
      Send(buf);
    }
!
    /**
     * Sends an integer to the back end in reverse order.
     *
--- 83,89 ----
        }
      Send(buf);
    }
!
    /**
     * Sends an integer to the back end in reverse order.
     *
***************
*** 106,112 ****
        }
      Send(buf);
    }
!
    /**
     * Send an array of bytes to the backend
     *
--- 107,113 ----
        }
      Send(buf);
    }
!
    /**
     * Send an array of bytes to the backend
     *
***************
*** 117,123 ****
    {
      pg_output.write(buf);
    }
!
    /**
     * Send an exact array of bytes to the backend - if the length
     * has not been reached, send nulls until it has.
--- 118,124 ----
    {
      pg_output.write(buf);
    }
!
    /**
     * Send an exact array of bytes to the backend - if the length
     * has not been reached, send nulls until it has.
***************
*** 130,136 ****
    {
      Send(buf,0,siz);
    }
!
    /**
     * Send an exact array of bytes to the backend - if the length
     * has not been reached, send nulls until it has.
--- 131,137 ----
    {
      Send(buf,0,siz);
    }
!
    /**
     * Send an exact array of bytes to the backend - if the length
     * has not been reached, send nulls until it has.
***************
*** 143,149 ****
    public void Send(byte buf[], int off, int siz) throws IOException
    {
      int i;
!
      pg_output.write(buf, off, ((buf.length-off) < siz ? (buf.length-off) : siz));
      if((buf.length-off) < siz)
        {
--- 144,150 ----
    public void Send(byte buf[], int off, int siz) throws IOException
    {
      int i;
!
      pg_output.write(buf, off, ((buf.length-off) < siz ? (buf.length-off) : siz));
      if((buf.length-off) < siz)
        {
***************
*** 153,159 ****
        }
        }
    }
!
    /**
     * Sends a packet, prefixed with the packet's length
     * @param buf buffer to send
--- 154,160 ----
        }
        }
    }
!
    /**
     * Sends a packet, prefixed with the packet's length
     * @param buf buffer to send
***************
*** 164,170 ****
      SendInteger(buf.length+4,4);
      Send(buf);
    }
!
    /**
     * Receives a single character from the backend
     *
--- 165,171 ----
      SendInteger(buf.length+4,4);
      Send(buf);
    }
!
    /**
     * Receives a single character from the backend
     *
***************
*** 174,180 ****
    public int ReceiveChar() throws SQLException
    {
      int c = 0;
!
      try
        {
      c = pg_input.read();
--- 175,181 ----
    public int ReceiveChar() throws SQLException
    {
      int c = 0;
!
      try
        {
      c = pg_input.read();
***************
*** 184,190 ****
        }
        return c;
    }
!
    /**
     * Receives an integer from the backend
     *
--- 185,191 ----
        }
        return c;
    }
!
    /**
     * Receives an integer from the backend
     *
***************
*** 195,207 ****
    public int ReceiveInteger(int siz) throws SQLException
    {
      int n = 0;
!
      try
        {
      for (int i = 0 ; i < siz ; i++)
        {
          int b = pg_input.read();
!
          if (b < 0)
            throw new PSQLException("postgresql.stream.eof");
          n = n | (b << (8 * i)) ;
--- 196,208 ----
    public int ReceiveInteger(int siz) throws SQLException
    {
      int n = 0;
!
      try
        {
      for (int i = 0 ; i < siz ; i++)
        {
          int b = pg_input.read();
!
          if (b < 0)
            throw new PSQLException("postgresql.stream.eof");
          n = n | (b << (8 * i)) ;
***************
*** 211,217 ****
        }
        return n;
    }
!
    /**
     * Receives an integer from the backend
     *
--- 212,218 ----
        }
        return n;
    }
!
    /**
     * Receives an integer from the backend
     *
***************
*** 222,234 ****
    public int ReceiveIntegerR(int siz) throws SQLException
    {
      int n = 0;
!
      try
        {
      for (int i = 0 ; i < siz ; i++)
        {
          int b = pg_input.read();
!
          if (b < 0)
            throw new PSQLException("postgresql.stream.eof");
          n = b | (n << 8);
--- 223,235 ----
    public int ReceiveIntegerR(int siz) throws SQLException
    {
      int n = 0;
!
      try
        {
      for (int i = 0 ; i < siz ; i++)
        {
          int b = pg_input.read();
!
          if (b < 0)
            throw new PSQLException("postgresql.stream.eof");
          n = b | (n << 8);
***************
*** 270,293 ****
      byte[] rst = bytePoolDim1.allocByte(maxsiz);
      return ReceiveString(rst, maxsiz, encoding);
    }
!
    /**
     * Receives a null-terminated string from the backend.  Maximum of
     * maxsiz bytes - if we don't see a null, then we assume something
     * has gone wrong.
     *
!    * @param rst byte array to read the String into. rst.length must
!    *        equal to or greater than maxsize.
     * @param maxsiz maximum length of string in bytes
     * @param encoding the charset encoding to use.
     * @return string from back end
     * @exception SQLException if an I/O error occurs
     */
!   public String ReceiveString(byte rst[], int maxsiz, String encoding)
        throws SQLException
    {
      int s = 0;
!
      try
        {
      while (s < maxsiz)
--- 271,294 ----
      byte[] rst = bytePoolDim1.allocByte(maxsiz);
      return ReceiveString(rst, maxsiz, encoding);
    }
!
    /**
     * Receives a null-terminated string from the backend.  Maximum of
     * maxsiz bytes - if we don't see a null, then we assume something
     * has gone wrong.
     *
!    * @param rst byte array to read the String into. rst.length must
!    *        equal to or greater than maxsize.
     * @param maxsiz maximum length of string in bytes
     * @param encoding the charset encoding to use.
     * @return string from back end
     * @exception SQLException if an I/O error occurs
     */
!   public String ReceiveString(byte rst[], int maxsiz, String encoding)
        throws SQLException
    {
      int s = 0;
!
      try
        {
      while (s < maxsiz)
***************
*** 318,324 ****
        }
        return v;
    }
!
    /**
     * Read a tuple from the back end.  A tuple is a two dimensional
     * array of bytes
--- 319,325 ----
        }
        return v;
    }
!
    /**
     * Read a tuple from the back end.  A tuple is a two dimensional
     * array of bytes
***************
*** 334,343 ****
      int i, bim = (nf + 7)/8;
      byte[] bitmask = Receive(bim);
      byte[][] answer = bytePoolDim2.allocByte(nf);
!
      int whichbit = 0x80;
      int whichbyte = 0;
!
      for (i = 0 ; i < nf ; ++i)
        {
      boolean isNull = ((bitmask[whichbyte] & whichbit) == 0);
--- 335,344 ----
      int i, bim = (nf + 7)/8;
      byte[] bitmask = Receive(bim);
      byte[][] answer = bytePoolDim2.allocByte(nf);
!
      int whichbit = 0x80;
      int whichbyte = 0;
!
      for (i = 0 ; i < nf ; ++i)
        {
      boolean isNull = ((bitmask[whichbyte] & whichbit) == 0);
***************
*** 347,367 ****
          ++whichbyte;
          whichbit = 0x80;
        }
!     if (isNull)
        answer[i] = null;
      else
        {
          int len = ReceiveIntegerR(4);
!         if (!bin)
            len -= 4;
!         if (len < 0)
            len = 0;
          answer[i] = Receive(len);
        }
        }
      return answer;
    }
!
    /**
     * Reads in a given number of bytes from the backend
     *
--- 348,368 ----
          ++whichbyte;
          whichbit = 0x80;
        }
!     if (isNull)
        answer[i] = null;
      else
        {
          int len = ReceiveIntegerR(4);
!         if (!bin)
            len -= 4;
!         if (len < 0)
            len = 0;
          answer[i] = Receive(len);
        }
        }
      return answer;
    }
!
    /**
     * Reads in a given number of bytes from the backend
     *
***************
*** 375,381 ****
      Receive(answer,0,siz);
      return answer;
    }
!
    /**
     * Reads in a given number of bytes from the backend
     *
--- 376,382 ----
      Receive(answer,0,siz);
      return answer;
    }
!
    /**
     * Reads in a given number of bytes from the backend
     *
***************
*** 387,394 ****
    public void Receive(byte[] b,int off,int siz) throws SQLException
    {
      int s = 0;
!
!     try
        {
      while (s < siz)
        {
--- 388,395 ----
    public void Receive(byte[] b,int off,int siz) throws SQLException
    {
      int s = 0;
!
!     try
        {
      while (s < siz)
        {
***************
*** 401,407 ****
        throw new PSQLException("postgresql.stream.ioerror",e);
        }
    }
!
    /**
     * This flushes any pending output to the backend. It is used primarily
     * by the Fastpath code.
--- 402,408 ----
        throw new PSQLException("postgresql.stream.ioerror",e);
        }
    }
!
    /**
     * This flushes any pending output to the backend. It is used primarily
     * by the Fastpath code.
***************
*** 415,421 ****
        throw new PSQLException("postgresql.stream.flush",e);
      }
    }
!
    /**
     * Closes the connection
     *
--- 416,422 ----
        throw new PSQLException("postgresql.stream.flush",e);
      }
    }
!
    /**
     * Closes the connection
     *
***************
*** 430,577 ****
      connection.close();
    }

-   /**
-    * Deallocate all resources that has been associated with any previous
-    * query.
-    */
-   public void deallocate(){
-       bytePoolDim1.deallocate();
-       bytePoolDim2.deallocate();
-   }
- }
-
- /**
-  * A simple and fast object pool implementation that can pool objects
-  * of any type. This implementation is not thread safe, it is up to the users
-  * of this class to assure thread safety.
-  */
- class ObjectPool {
-     int cursize = 0;
-     int maxsize = 16;
-     Object arr[] = new Object[maxsize];
-
-     public void add(Object o){
-     if(cursize >= maxsize){
-         Object newarr[] = new Object[maxsize*2];
-         System.arraycopy(arr, 0, newarr, 0, maxsize);
-         maxsize = maxsize * 2;
-         arr = newarr;
-     }
-     arr[cursize++] = o;
-     }
-
-     public Object remove(){
-     return arr[--cursize];
-     }
-     public boolean isEmpty(){
-     return cursize == 0;
-     }
-     public int size(){
-     return cursize;
-     }
-     public void addAll(ObjectPool pool){
-     int srcsize = pool.size();
-     if(srcsize == 0)
-         return;
-     int totalsize = srcsize + cursize;
-     if(totalsize > maxsize){
-         Object newarr[] = new Object[totalsize*2];
-         System.arraycopy(arr, 0, newarr, 0, cursize);
-         maxsize = maxsize = totalsize * 2;
-         arr = newarr;
-     }
-     System.arraycopy(pool.arr, 0, arr, cursize, srcsize);
-     cursize = totalsize;
-     }
-     public void clear(){
-         cursize = 0;
-     }
  }

- /**
-  * A simple and efficient class to pool one dimensional byte arrays
-  * of different sizes.
-  */
- class BytePoolDim1 {
-     int maxsize = 256;
-     ObjectPool notusemap[] = new ObjectPool[maxsize + 1];
-     ObjectPool inusemap[] = new ObjectPool[maxsize + 1];
-
-     public BytePoolDim1(){
-     for(int i = 0; i <= maxsize; i++){
-         inusemap[i] = new ObjectPool();
-         notusemap[i] = new ObjectPool();
-     }
-     }
-
-     public byte[] allocByte(int size){
-     if(size > maxsize){
-         return new byte[size];
-     }
-
-     ObjectPool not_usel = notusemap[size];
-     ObjectPool in_usel = inusemap[size];
-     byte b[] = null;
-
-     if(!not_usel.isEmpty()) {
-         Object o = not_usel.remove();
-         b = (byte[]) o;
-     } else
-         b = new byte[size];
-     in_usel.add(b);
-
-     return b;
-     }
-
-     public void deallocate(){
-     for(int i = 0; i <= maxsize; i++){
-         notusemap[i].addAll(inusemap[i]);
-         inusemap[i].clear();
-     }
-
-     }
- }
-
-
-
- /**
-  * A simple and efficient class to pool two dimensional byte arrays
-  * of different sizes.
-  */
- class BytePoolDim2 {
-     int maxsize = 32;
-     ObjectPool notusemap[] = new ObjectPool[maxsize + 1];
-     ObjectPool inusemap[] = new ObjectPool[maxsize + 1];
-
-     public BytePoolDim2(){
-     for(int i = 0; i <= maxsize; i++){
-         inusemap[i] = new ObjectPool();
-         notusemap[i] = new ObjectPool();
-     }
-     }
-
-     public byte[][] allocByte(int size){
-     if(size > maxsize){
-         return new byte[size][0];
-     }
-     ObjectPool not_usel = notusemap[size];
-     ObjectPool in_usel =  inusemap[size];
-
-     byte b[][] = null;
-
-     if(!not_usel.isEmpty()) {
-         Object o = not_usel.remove();
-         b = (byte[][]) o;
-     } else
-         b = new byte[size][0];
-     in_usel.add(b);
-     return b;
-     }
-
-     public void deallocate(){
-     for(int i = 0; i <= maxsize; i++){
-         notusemap[i].addAll(inusemap[i]);
-         inusemap[i].clear();
-     }
-     }
- }
--- 431,435 ----