Re: improvements for dict_xsyn extended synonym dictionary - RRR

Andres Freund <andres@anarazel.de> writes:

Hi Andres,

Thank you for review of my patch.

> Some points:
> - Patch looks generally sound
> - lacks a bit of a motivational statement, even though one can imagine uses

The patch has initially been motivated by the request in pgsql-general
(http://archives.postgresql.org/pgsql-general/2009-02/msg00102.php).

> - Imho mode=MAP should error out if keeporig is false
> - I personally find the the names for the different modes a bit nondescriptive.
>   One possibility would be to introduce parameters like:
>     - matchorig
>     - matchsynonym
>     - keeporig
>     - keepsynonym
> That sounds way much easier to grasp for me.

Yes, I agree. In such a way user has the complete (and more straightforward)
control over the dictionary behaviour.

Here is the revised patch version, with following options:

     * matchorig controls whether the original word is accepted by the
       dictionary. Default is true.

     * keeporig controls whether the original word is included (if true)
       in results, or only its synonyms (if false). Default is true.

     * matchsynonyms controls whether any of the synonyms is accepted by
       the dictionary (if true). Default is false.

     * keepsynonyms controls whether synonyms are returned by the
       dictionary (if true). Default is true.

Defaults are set to keep default behaviour compatible with original version.

Thanks,
Sergey

Index: contrib/dict_xsyn/dict_xsyn.c
===================================================================
RCS file: /projects/cvsroot/pgsql/contrib/dict_xsyn/dict_xsyn.c,v
retrieving revision 1.6
diff -u -r1.6 dict_xsyn.c
--- contrib/dict_xsyn/dict_xsyn.c    1 Jan 2009 17:23:32 -0000    1.6
+++ contrib/dict_xsyn/dict_xsyn.c    27 Jul 2009 09:51:52 -0000
@@ -26,6 +26,7 @@
     char       *key;            /* Word */
     char       *value;            /* Unparsed list of synonyms, including the
                                  * word itself */
+    int         pos;            /* Position of key word in original string */
 } Syn;

 typedef struct
@@ -33,7 +34,11 @@
     int            len;
     Syn           *syn;

+    bool        matchorig;
     bool        keeporig;
+    bool        matchsynonyms;
+    bool        keepsynonyms;
+
 } DictSyn;


@@ -88,6 +93,7 @@
     {
         char       *value;
         char       *key;
+        char       *pos;
         char       *end = NULL;

         if (*line == '\0')
@@ -96,26 +102,39 @@
         value = lowerstr(line);
         pfree(line);

-        key = find_word(value, &end);
-        if (!key)
-        {
-            pfree(value);
-            continue;
-        }
+        pos = value;

-        if (cur == d->len)
+        while((key = find_word(pos, &end)) != NULL)
         {
-            d->len = (d->len > 0) ? 2 * d->len : 16;
-            if (d->syn)
-                d->syn = (Syn *) repalloc(d->syn, sizeof(Syn) * d->len);
-            else
-                d->syn = (Syn *) palloc(sizeof(Syn) * d->len);
-        }
+            if (cur == d->len)
+            {
+                d->len = (d->len > 0) ? 2 * d->len : 16;
+                if (d->syn)
+                    d->syn = (Syn *) repalloc(d->syn, sizeof(Syn) * d->len);
+                else
+                    d->syn = (Syn *) palloc(sizeof(Syn) * d->len);
+            }

-        d->syn[cur].key = pnstrdup(key, end - key);
-        d->syn[cur].value = value;
+            /* Read first word only if we will match it */
+            if (pos != value || d->matchorig)
+            {
+                d->syn[cur].key = pnstrdup(key, end - key);
+                d->syn[cur].value = pstrdup(value);
+                d->syn[cur].pos = key - value;
+
+                cur++;
+            }

-        cur++;
+            pos = end;
+
+            /* Don't read synonyms if we do not match them */
+            if (!d->matchsynonyms)
+            {
+                break;
+            }
+        }
+
+        pfree(value);
     }

     tsearch_readline_end(&trst);
@@ -133,23 +152,40 @@
     List       *dictoptions = (List *) PG_GETARG_POINTER(0);
     DictSyn    *d;
     ListCell   *l;
-
+    char       *filename = NULL;
+
     d = (DictSyn *) palloc0(sizeof(DictSyn));
     d->len = 0;
     d->syn = NULL;
+    d->matchorig = true;
     d->keeporig = true;
-
+    d->matchsynonyms = false;
+    d->keepsynonyms = true;
+
     foreach(l, dictoptions)
     {
         DefElem    *defel = (DefElem *) lfirst(l);

-        if (pg_strcasecmp(defel->defname, "KEEPORIG") == 0)
+        if (pg_strcasecmp(defel->defname, "MATCHORIG") == 0)
+        {
+            d->matchorig = defGetBoolean(defel);
+        }
+        else if (pg_strcasecmp(defel->defname, "KEEPORIG") == 0)
         {
             d->keeporig = defGetBoolean(defel);
         }
+        else if (pg_strcasecmp(defel->defname, "MATCHSYNONYMS") == 0)
+        {
+            d->matchsynonyms = defGetBoolean(defel);
+        }
+        else if (pg_strcasecmp(defel->defname, "KEEPSYNONYMS") == 0)
+        {
+            d->keepsynonyms = defGetBoolean(defel);
+        }
         else if (pg_strcasecmp(defel->defname, "RULES") == 0)
         {
-            read_dictionary(d, defGetString(defel));
+            /* we can't read the rules before parsing all options! */
+            filename = pstrdup(defGetString(defel));
         }
         else
         {
@@ -160,6 +196,12 @@
         }
     }

+    if(filename)
+    {
+        read_dictionary(d, filename);
+        pfree(filename);
+    }
+
     PG_RETURN_POINTER(d);
 }

@@ -198,7 +240,6 @@
         int            value_length = strlen(value);
         char       *pos = value;
         int            nsyns = 0;
-        bool        is_first = true;

         res = palloc(0);

@@ -214,8 +255,8 @@
             res = repalloc(res, sizeof(TSLexeme) * (nsyns + 2));
             res[nsyns].lexeme = NULL;

-            /* first word is added to result only if KEEPORIG flag is set */
-            if (d->keeporig || !is_first)
+            /* The first word is added only if keeporig=true */
+            if (pos != value || d->keeporig)
             {
                 res[nsyns].lexeme = pstrdup(syn);
                 res[nsyns + 1].lexeme = NULL;
@@ -223,9 +264,12 @@
                 nsyns++;
             }

-            is_first = false;
-
             pos = end + 1;
+
+            if(!d->keepsynonyms)
+            {
+                break;
+            }
         }

         pfree(value);
Index: contrib/dict_xsyn/expected/dict_xsyn.out
===================================================================
RCS file: /projects/cvsroot/pgsql/contrib/dict_xsyn/expected/dict_xsyn.out,v
retrieving revision 1.1
diff -u -r1.1 dict_xsyn.out
--- contrib/dict_xsyn/expected/dict_xsyn.out    15 Oct 2007 21:36:50 -0000    1.1
+++ contrib/dict_xsyn/expected/dict_xsyn.out    27 Jul 2009 09:51:53 -0000
@@ -5,10 +5,76 @@
 SET client_min_messages = warning;
 \set ECHO none
 RESET client_min_messages;
---configuration
-ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false);
+-- default configuration - match first word and return it among with all synonyms
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=true, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=false);
 --lexize
 SELECT ts_lexize('xsyn', 'supernova');
+        ts_lexize
+--------------------------
+ {supernova,sn,sne,1987a}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'sn');
+ ts_lexize
+-----------
+
+(1 row)
+
+SELECT ts_lexize('xsyn', 'grb');
+ ts_lexize
+-----------
+
+(1 row)
+
+-- the same, but return only synonyms
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=false);
+SELECT ts_lexize('xsyn', 'supernova');
+   ts_lexize
+----------------
+ {sn,sne,1987a}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'sn');
+ ts_lexize
+-----------
+
+(1 row)
+
+SELECT ts_lexize('xsyn', 'grb');
+ ts_lexize
+-----------
+
+(1 row)
+
+-- match any word and return all words
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=true, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+        ts_lexize
+--------------------------
+ {supernova,sn,sne,1987a}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'sn');
+        ts_lexize
+--------------------------
+ {supernova,sn,sne,1987a}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'grb');
+ ts_lexize
+-----------
+
+(1 row)
+
+-- match any word and return all words except first one
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+   ts_lexize
+----------------
+ {sn,sne,1987a}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'sn');
    ts_lexize
 ----------------
  {sn,sne,1987a}
@@ -20,3 +86,63 @@

 (1 row)

+-- match any synonym but not first word, and return first word instead
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=true, MATCHORIG=false, KEEPSYNONYMS=false,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+ ts_lexize
+-----------
+
+(1 row)
+
+SELECT ts_lexize('xsyn', 'sn');
+  ts_lexize
+-------------
+ {supernova}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'grb');
+ ts_lexize
+-----------
+
+(1 row)
+
+-- do not match or return anything
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=false, KEEPSYNONYMS=false,
MATCHSYNONYMS=false);
+SELECT ts_lexize('xsyn', 'supernova');
+ ts_lexize
+-----------
+
+(1 row)
+
+SELECT ts_lexize('xsyn', 'sn');
+ ts_lexize
+-----------
+
+(1 row)
+
+SELECT ts_lexize('xsyn', 'grb');
+ ts_lexize
+-----------
+
+(1 row)
+
+-- match any word but return nothing
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=true, KEEPSYNONYMS=false,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+ ts_lexize
+-----------
+ {}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'sn');
+ ts_lexize
+-----------
+ {}
+(1 row)
+
+SELECT ts_lexize('xsyn', 'grb');
+ ts_lexize
+-----------
+
+(1 row)
+
Index: contrib/dict_xsyn/sql/dict_xsyn.sql
===================================================================
RCS file: /projects/cvsroot/pgsql/contrib/dict_xsyn/sql/dict_xsyn.sql,v
retrieving revision 1.1
diff -u -r1.1 dict_xsyn.sql
--- contrib/dict_xsyn/sql/dict_xsyn.sql    15 Oct 2007 21:36:50 -0000    1.1
+++ contrib/dict_xsyn/sql/dict_xsyn.sql    27 Jul 2009 09:51:53 -0000
@@ -8,9 +8,47 @@
 \set ECHO all
 RESET client_min_messages;

---configuration
-ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false);
+-- default configuration - match first word and return it among with all synonyms
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=true, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=false);

 --lexize
 SELECT ts_lexize('xsyn', 'supernova');
+SELECT ts_lexize('xsyn', 'sn');
 SELECT ts_lexize('xsyn', 'grb');
+
+-- the same, but return only synonyms
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=false);
+SELECT ts_lexize('xsyn', 'supernova');
+SELECT ts_lexize('xsyn', 'sn');
+SELECT ts_lexize('xsyn', 'grb');
+
+-- match any word and return all words
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=true, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+SELECT ts_lexize('xsyn', 'sn');
+SELECT ts_lexize('xsyn', 'grb');
+
+-- match any word and return all words except first one
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=true, KEEPSYNONYMS=true,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+SELECT ts_lexize('xsyn', 'sn');
+SELECT ts_lexize('xsyn', 'grb');
+
+-- match any synonym but not first word, and return first word instead
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=true, MATCHORIG=false, KEEPSYNONYMS=false,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+SELECT ts_lexize('xsyn', 'sn');
+SELECT ts_lexize('xsyn', 'grb');
+
+-- do not match or return anything
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=false, KEEPSYNONYMS=false,
MATCHSYNONYMS=false);
+SELECT ts_lexize('xsyn', 'supernova');
+SELECT ts_lexize('xsyn', 'sn');
+SELECT ts_lexize('xsyn', 'grb');
+
+-- match any word but return nothing
+ALTER TEXT SEARCH DICTIONARY xsyn (RULES='xsyn_sample', KEEPORIG=false, MATCHORIG=true, KEEPSYNONYMS=false,
MATCHSYNONYMS=true);
+SELECT ts_lexize('xsyn', 'supernova');
+SELECT ts_lexize('xsyn', 'sn');
+SELECT ts_lexize('xsyn', 'grb');
+
Index: doc/src/sgml/dict-xsyn.sgml
===================================================================
RCS file: /projects/cvsroot/pgsql/doc/src/sgml/dict-xsyn.sgml,v
retrieving revision 1.2
diff -u -r1.2 dict-xsyn.sgml
--- doc/src/sgml/dict-xsyn.sgml    6 Dec 2007 04:12:10 -0000    1.2
+++ doc/src/sgml/dict-xsyn.sgml    27 Jul 2009 09:51:56 -0000
@@ -23,13 +23,32 @@
   <itemizedlist>
    <listitem>
     <para>
+     <literal>matchorig</> controls whether the original word is accepted by
+     the dictionary. Default is <literal>true</>.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
      <literal>keeporig</> controls whether the original word is included (if
-     <literal>true</>), or only its synonyms (if <literal>false</>). Default
-     is <literal>true</>.
+     <literal>true</>) in results, or only its synonyms (if
+     <literal>false</>). Default is <literal>true</>.
+    </para>
+   </listitem>
+   <listitem>
+    <para>
+     <literal>matchsynonyms</> controls whether any of the synonyms is accepted
+     by the dictionary (if <literal>true</>). Default is <literal>false</>.
     </para>
    </listitem>
    <listitem>
     <para>
+     <literal>keepsynonyms</> controls whether synonyms are returned by the
+     dictionary (if <literal>true</>). Default is <literal>true</>.
+    </para>
+   </listitem>
+
+   <listitem>
+    <para>
      <literal>rules</> is the base name of the file containing the list of
      synonyms.  This file must be stored in
      <filename>$SHAREDIR/tsearch_data/</> (where <literal>$SHAREDIR</> means
@@ -90,7 +109,31 @@
 mydb=# SELECT ts_lexize('xsyn', 'word');
       ts_lexize
 -----------------------
+ {syn1,syn2,syn3}
+
+mydb# ALTER TEXT SEARCH DICTIONARY xsyn (RULES='my_rules', KEEPORIG=true);
+ALTER TEXT SEARCH DICTIONARY
+
+mydb=# SELECT ts_lexize('xsyn', 'word');
+      ts_lexize
+-----------------------
  {word,syn1,syn2,syn3}
+
+mydb# ALTER TEXT SEARCH DICTIONARY xsyn (RULES='my_rules', KEEPORIG=false, MATCHSYNONYMS=true);
+ALTER TEXT SEARCH DICTIONARY
+
+mydb=# SELECT ts_lexize('xsyn', 'syn1');
+      ts_lexize
+-----------------------
+ {syn1,syn2,syn3}
+
+mydb# ALTER TEXT SEARCH DICTIONARY xsyn (RULES='my_rules', KEEPORIG=true, MATCHORIG=false, KEEPSYNONYMS=false);
+ALTER TEXT SEARCH DICTIONARY
+
+mydb=# SELECT ts_lexize('xsyn', 'syn1');
+      ts_lexize
+-----------------------
+ {word}
 </programlisting>

    but real-world usage will involve including it in a text search