New Full Text Index using contrib/fulltextindex which now able to processing Traditional Chinese characters(Big5 encoding)

Hi, all, I found that contrib/fulltextindex is unable to process multibyte
characters, so I tried to make it suit for my mother language as Chinese, I
believe it's able to process Unicode by wcrok(), but I'm lazy to complete
it.;)

USAGE:

Just replace the fti.c from this mail on contrib/fulltextindex and re-make
it.

And the indexing/query way is most the same.

CODE:
/* The difference: breakup() and issleadbyte()

#include "postgres.h"

#include <ctype.h>

#include "executor/spi.h"
#include "commands/trigger.h"

/*
 * Trigger function accepts variable number of arguments:
 *
 *  1. relation in which to store the substrings
 *  2. fields to extract substrings from
 *
 * The relation in which to insert *must* have the following layout:
 *
 *  string  varchar(#)
 *  id   oid
 *
 *  where # is the largest size of the varchar columns being indexed
 *
 * Example:
 *
 * -- Create the SQL function based on the compiled shared object
 * create function fti() returns opaque as
 *   '/usr/local/pgsql/lib/contrib/fti.so' language 'C';
 *
 * -- Create the FTI table
 * create table product_fti (string varchar(255), id oid);
 *
 * -- Create an index to assist string matches
 * create index product_fti_string_idx on product_fti (string);
 *
 * -- Create an index to assist trigger'd deletes
 * create index product_fti_id_idx on product_fti (id);
 *
 * -- Create an index on the product oid column to assist joins
 * -- between the fti table and the product table
 * create index product_oid_idx on product (oid);
 *
 * -- Create the trigger to perform incremental changes to the full text
index.
 * create trigger product_fti_trig after update or insert or delete on
product
 * for each row execute procedure fti(product_fti, title, artist);
 *            ^^^^^^^^^^^
 *            table where full text index is stored
 *             ^^^^^^^^^^^^^
 *             columns to index in the base table
 *
 * After populating 'product', try something like:
 *
 * SELECT DISTINCT(p.*) FROM product p, product_fti f1, product_fti f2 WHERE
 * f1.string ~ '^slippery' AND f2.string ~ '^wet' AND p.oid=f1.id AND
p.oid=f2.id;
 *
 * To check that your indicies are being used correctly, make sure you
 * EXPLAIN SELECT ... your test query above.
 *
 * CHANGELOG
 * ---------
 *
 * august 3 2001
 *     Extended fti function to accept more than one column as a
 *     parameter and all specified columns are indexed.  Changed
 *     all uses of sprintf to snprintf.  Made error messages more
 *     consistent.
 *
 * march 4 1998 Changed breakup() to return less substrings. Only breakup
 *     in word parts which are in turn shortened from the start
 *     of the word (ie. word, ord, rd)
 *     Did allocation of substring buffer outside of breakup()
 *
 * oct. 5 1997, fixed a bug in string breakup (where there are more nonalpha
 *     characters between words then 1).
 *
 * oct 4-5 1997 implemented the thing, at least the basic functionallity
 *     of it all....
 *
 * TODO
 * ----
 *
 *  prevent generating duplicate words for an oid in the fti table
 *  save a plan for deletes
 *  create a function that will make the index *after* we have populated
 *  the main table (probably first delete all contents to be sure there's
 *  nothing in it, then re-populate the fti-table)
 *
 *  can we do something with operator overloading or a seperate function
 *  that can build the final query automatigally?
 */

#define MAX_FTI_QUERY_LENGTH 8192

extern Datum fti(PG_FUNCTION_ARGS);
static char *breakup(char *, char *);
static bool is_stopword(char *);
static bool isleadbyte(unsigned char ch);
static bool new_tuple = false;


#ifdef USE_STOP_WORDS

/* THIS LIST MUST BE IN SORTED ORDER, A BINARY SEARCH IS USED!!!! */
char    *StopWords[] = {  /* list of words to skip in indexing */
 "no",
 "the",
 "yes"
};
#endif   /* USE_STOP_WORDS */

/* stuff for caching query-plans, stolen from contrib/spi/\*.c */
typedef struct
{
 char    *ident;
 int   nplans;
 void   **splan;
} EPlan;

static EPlan *InsertPlans = NULL;
static EPlan *DeletePlans = NULL;
static int nInsertPlans = 0;
static int nDeletePlans = 0;

static EPlan *find_plan(char *ident, EPlan ** eplan, int *nplans);

/***********************************************************************/
PG_FUNCTION_INFO_V1(fti);

Datum
fti(PG_FUNCTION_ARGS)
{
 TriggerData *trigdata;
 Trigger    *trigger;  /* to get trigger name */
 int   nargs;   /* # of arguments */
 char   **args;   /* arguments */
 char    *relname;  /* triggered relation name */
 Relation rel;   /* triggered relation */
 char    *indexname;  /* name of table for substrings */
 HeapTuple rettuple = NULL;
 TupleDesc tupdesc;  /* tuple description */
 bool  isinsert = false;
 bool  isdelete = false;
 int   ret;
 char  query[MAX_FTI_QUERY_LENGTH];
 Oid   oid;

 /*
  * FILE   *debug;
  */

 /*
  * debug = fopen("/dev/xconsole", "w"); fprintf(debug, "FTI: entered
  * function\n"); fflush(debug);
  */

 if (!CALLED_AS_TRIGGER(fcinfo))
  elog(ERROR, "Full Text Indexing: Not fired by trigger manager");

 /* It's safe to cast now that we've checked */
 trigdata = (TriggerData *) fcinfo->context;

 if (TRIGGER_FIRED_FOR_STATEMENT(trigdata->tg_event))
  elog(ERROR, "Full Text Indexing: Can't process STATEMENT events");
 if (TRIGGER_FIRED_BEFORE(trigdata->tg_event))
  elog(ERROR, "Full Text Indexing: Must be fired AFTER event");

 if (TRIGGER_FIRED_BY_INSERT(trigdata->tg_event))
  isinsert = true;
 if (TRIGGER_FIRED_BY_UPDATE(trigdata->tg_event))
 {
  isdelete = true;
  isinsert = true;
 }
 if (TRIGGER_FIRED_BY_DELETE(trigdata->tg_event))
  isdelete = true;

 trigger = trigdata->tg_trigger;
 rel = trigdata->tg_relation;
 relname = SPI_getrelname(rel);
 rettuple = trigdata->tg_trigtuple;
 if (isdelete && isinsert) /* is an UPDATE */
  rettuple = trigdata->tg_newtuple;

       if ((ret = SPI_connect()) < 0)
  elog(ERROR, "Full Text Indexing: SPI_connect: Failed, returned %d\n",
ret);

 nargs = trigger->tgnargs;
 if (nargs < 2)
  elog(ERROR, "Full Text Indexing: Trigger must have at least 2
arguments\n");

 args = trigger->tgargs;
 indexname = args[0];
 tupdesc = rel->rd_att;  /* what the tuple looks like (?) */

 /* get oid of current tuple, needed by all, so place here */
 oid = rettuple->t_data->t_oid;
 if (!OidIsValid(oid))
  elog(ERROR, "Full Text Indexing: Oid of current tuple is invalid");

 if (isdelete)
 {
  void    *pplan;
  Oid     *argtypes;
  Datum  values[1];
  EPlan    *plan;
  int   i;

  snprintf(query, MAX_FTI_QUERY_LENGTH, "D%s", indexname);
  for (i = 1; i < nargs; i++)
   snprintf(query, MAX_FTI_QUERY_LENGTH, "%s$%s", query, args[i]);

  plan = find_plan(query, &DeletePlans, &nDeletePlans);
  if (plan->nplans <= 0)
  {
   argtypes = (Oid *) palloc(sizeof(Oid));

   argtypes[0] = OIDOID;

   snprintf(query, MAX_FTI_QUERY_LENGTH, "DELETE FROM %s WHERE id = $1",
indexname);
   pplan = SPI_prepare(query, 1, argtypes);
   if (!pplan)
    elog(ERROR, "Full Text Indexing: SPI_prepare: Returned NULL in delete");
   pplan = SPI_saveplan(pplan);
   if (pplan == NULL)
    elog(ERROR, "Full Text Indexing: SPI_saveplan: Returned NULL in
delete");

   plan->splan = (void **) malloc(sizeof(void *));
   *(plan->splan) = pplan;
   plan->nplans = 1;
  }

  values[0] = oid;

  ret = SPI_execp(*(plan->splan), values, NULL, 0);
  if (ret != SPI_OK_DELETE)
   elog(ERROR, "Full Text Indexing: SPI_execp: Error executing plan in
delete");
 }

 if (isinsert)
 {
  char    *substring;
  char    *column;
  void    *pplan;
  Oid     *argtypes;
  Datum  values[2];
  int   colnum;
  struct varlena *data;
  EPlan    *plan;
  int   i;
  char    *buff;
  char    *string;

  snprintf(query, MAX_FTI_QUERY_LENGTH, "I%s", indexname);
  for (i = 1; i < nargs; i++)
   snprintf(query, MAX_FTI_QUERY_LENGTH, "%s$%s", query, args[i]);

  plan = find_plan(query, &InsertPlans, &nInsertPlans);

  /* no plan yet, so allocate mem for argtypes */
  if (plan->nplans <= 0)
  {
   argtypes = (Oid *) palloc(2 * sizeof(Oid));

   argtypes[0] = VARCHAROID; /* create table t_name (string
           * varchar, */
   argtypes[1] = OIDOID;  /* id   oid);    */

   /* prepare plan to gain speed */
   snprintf(query, MAX_FTI_QUERY_LENGTH, "INSERT INTO %s (string, id) VALUES
($1, $2)",
      indexname);
   pplan = SPI_prepare(query, 2, argtypes);
   if (!pplan)
    elog(ERROR, "Full Text Indexing: SPI_prepare: Returned NULL in insert");

   pplan = SPI_saveplan(pplan);
   if (pplan == NULL)
    elog(ERROR, "Full Text Indexing: SPI_saveplan: Returned NULL in
insert");

   plan->splan = (void **) malloc(sizeof(void *));
   *(plan->splan) = pplan;
   plan->nplans = 1;
  }

  /* prepare plan for query */
  for (i = 0; i < nargs - 1; i++)
  {
   colnum = SPI_fnumber(tupdesc, args[i + 1]);
   if (colnum == SPI_ERROR_NOATTRIBUTE)
    elog(ERROR, "Full Text Indexing: SPI_fnumber: Column '%s' of '%s' not
found", args[i + 1], indexname);

   /* Get the char* representation of the column */
   column = SPI_getvalue(rettuple, tupdesc, colnum);

   /* make sure we don't try to index NULL's */
   if (column)
   {
    string = column;
    while (*string != '\0')
    {
     *string = tolower((unsigned char) *string);
     string++;
    }

    data = (struct varlena *) palloc(sizeof(int32) + strlen(column) +1);
    buff = palloc(strlen(column) + 1);
    /* saves lots of calls in while-loop and in breakup() */

    new_tuple = true;

    while ((substring = breakup(column, buff)))
    {
     int   l;

     l = strlen(substring);

     data->vl_len = l + sizeof(int32);
     memcpy(VARDATA(data), substring, l);
     values[0] = PointerGetDatum(data);
     values[1] = oid;

     ret = SPI_execp(*(plan->splan), values, NULL, 0);
     if (ret != SPI_OK_INSERT)
      elog(ERROR, "Full Text Indexing: SPI_execp: Error executing plan in
insert");
    }
    pfree(buff);
    pfree(data);
   }
  }
 }

 SPI_finish();
 return PointerGetDatum(rettuple);
}

static char *
breakup(char *string, char *substring)
{
 static char *last_start;
 static char *cur_pos;
 static char *string_end;

 if (new_tuple)
 {
  string_end = &string[strlen(string)-1];
  cur_pos = last_start = &string[0];
  new_tuple = false;/* don't initialize this next time */
 }

 while (cur_pos <= string_end)/* don't read after end of 'string' */
 {
  if (isleadbyte((unsigned)*cur_pos )) {
   /* Bingo, got a Big-5 word (2 bytes) */
   cur_pos += 2;
   memcpy(substring, last_start, cur_pos - last_start);
   substring[cur_pos - last_start] = '\0';
   if (!is_stopword(substring))
    return substring;
  } else if (isalnum((unsigned char) *cur_pos)) {
   /* Houston, we have a substring! :) */
   cur_pos++;
   memcpy(substring, last_start, cur_pos - last_start);
   substring[cur_pos - last_start] = '\0';
   if (!is_stopword(substring))
    return substring;
  } else {
   last_start = cur_pos + 1;
   cur_pos = last_start;
  }
 }

 return NULL;/* we've processed all of 'string' */
}

bool isleadbyte(unsigned char ch)
{
 if ((ch >= 0xA1) && (ch <= 0xFE))
  return true;

 if ((ch >= 0x8E) && (ch <= 0xA0))
  return true;

 if ((ch >= 0x81) && (ch <= 0x8D))
  return true;

return false;
}
/* copied from src/backend/parser/keywords.c and adjusted for our
situation*/
static bool
is_stopword(char *text)
{
#ifdef USE_STOP_WORDS
 char   **StopLow;  /* for list of stop-words */
 char   **StopHigh;
 char   **StopMiddle;
 int   difference;

 StopLow = &StopWords[0]; /* initialize stuff for binary search */
 StopHigh = endof(StopWords);

 /* Loop invariant: *StopLow <= text < *StopHigh */

 while (StopLow < StopHigh)
 {
  StopMiddle = StopLow + (StopHigh - StopLow) / 2;
  difference = strcmp(*StopMiddle, text);
  if (difference == 0)
   return (true);
  else if (difference < 0)
   StopLow = StopMiddle + 1;
  else
   StopHigh = StopMiddle;
 }
#endif   /* USE_STOP_WORDS */

 return (false);
}

/* for caching of query plans, stolen from contrib/spi/\*.c */
static EPlan *
find_plan(char *ident, EPlan ** eplan, int *nplans)
{
 EPlan    *newp;
 int   i;

 if (*nplans > 0)
 {
  for (i = 0; i < *nplans; i++)
  {
   if (strcmp((*eplan)[i].ident, ident) == 0)
    break;
  }
  if (i != *nplans)
   return (*eplan + i);
  *eplan = (EPlan *) realloc(*eplan, (i + 1) * sizeof(EPlan));
  newp = *eplan + i;
 }
 else
 {
  newp = *eplan = (EPlan *) malloc(sizeof(EPlan));
  (*nplans) = i = 0;
 }

 newp->ident = (char *) malloc(strlen(ident) + 1);
 strcpy(newp->ident, ident);
 newp->nplans = 0;
 newp->splan = NULL;
 (*nplans)++;

 return (newp);
}