Certain query eating up all free memory (out of memory error)

Hi group,

I could really use your help with this one. I don't have all the
details right now (I can provide more descriptions tomorrow and logs
if needed), but maybe this will be enough:

I have written a PG (8.3.8) module, which uses Flex Lexical Analyser.
It takes text from database field and finds matches for defined rules.
It returns a set of two text fields (value found and value type).

When I run query like this:
SELECT * FROM flex_me(SELECT some_text FROM some_table WHERE id = 1);
It works perfectly fine. Memory never reaches more than 1% (usually
its below 0.5% of system mem).

But when I run query like this:
SELECT flex_me(some_text_field) FROM some_table WHERE id = 1;
Memory usage goes through the roof, and if the result is over about
10k matches (rows) it eats up all memory and I get "out of memory"
error.

I try to free all memory allocated, and even did a version with double
linked list of results but the same behaviour persists. I tried to
track it down on my own and from my own trials it seems that the
problem lies directly in the set returning function in File 2
"flex_me()" as even with 40k of results in a 2 column array it
shouldn't take more than 1MB of memory. Also when I run it just to the
point of SRF_IS_FIRSTCALL() (whole bit) the memory usage doesn't go
up, but when subsequent SRF_PERCALL calls are made it's where the
memory usage goes through the roof.

Btw, if the following code contains some nasty errors and I'm pretty
sure it does, please know that I'm just learning PG and C programming.
Any help or tips would be greatly appreciated.

Simplified (but still relevant) code below:

File 1 (Flex parser template which is compiled with flex):

%{
#include <stdio.h>

extern void *addToken(int type);
extern char ***flexme(char *ptr);

#define T_NUM  1
#define S_NUM  "number"
#define T_FLO  2
#define S_FLO  "float"
#define T_DAT  3
#define S_DAT  "date
#define T_WRD  7
#define S_WRD  "word"

char ***vals;

int cnt = 0, mem_cnt = 64;

%}

DGT          [0-9]
NUMBER       (-)?{DGT}+
FLOAT        ((-)?{DGT}+[\.,]{DGT}+)|{NUMBER}

DATE_S1      "-"
DATE_S2      ","
DATE_S3      "."
DATE_S4      "/"
DATE_S5      ""
DATE_YY      ([0-9]|([0-9][0-9])|([0-1][0-9][0-9][0-9])|(2[0-4][0-9][0-9]))
DATE_DD      ([1-9]|(([0-2][0-9])|(3[0-1])))
DATE_MM      ([1-9]|((0[1-9])|(1[0-2])))

DATE_YMD_S1  ({DATE_YY}{DATE_S1}{DATE_MM}{DATE_S1}{DATE_DD})
DATE_YMD_S2  ({DATE_YY}{DATE_S2}{DATE_MM}{DATE_S2}{DATE_DD})
DATE_YMD_S3  ({DATE_YY}{DATE_S3}{DATE_MM}{DATE_S3}{DATE_DD})
DATE_YMD_S4  ({DATE_YY}{DATE_S4}{DATE_MM}{DATE_S4}{DATE_DD})
DATE_YMD_S5  ({DATE_YY}{DATE_S5}{DATE_MM}{DATE_S5}{DATE_DD})
DATE_YMD     ({DATE_YMD_S1}|{DATE_YMD_S2}|{DATE_YMD_S3}|{DATE_YMD_S4}|{DATE_YMD_S5})

WORD         ([a-zA-Z0-9]+)

%%

{FLOAT}      addToken(T_FLO);

{DATE_YMD}   addToken(T_DAT);

{WORD}       addToken(T_WRD);

.|\n     /* eat up any unmatched character */

%%

void *
addToken(int type)
{
  int   x = 0;

//    elog(NOTICE,"W[%d] %s", type, yytext);

    //check if we need to add more mem
    if (mem_cnt-1 <= cnt) {
        mem_cnt *= 2;
        vals = repalloc(vals, mem_cnt * sizeof(char *));
//        elog(NOTICE, "mem increased to: %d", mem_cnt*sizeof(char *));
    }
    vals[cnt] = palloc(2 * sizeof(char *));

    //types
    switch (type) {
        case T_FLO:    //float
            x = strlen(S_FLO);
            vals[cnt][1] = palloc((x+1) * sizeof(char));
            strncpy(vals[cnt][1], S_FLO, x);
            vals[cnt][1][x] = '\0';
            break;
        case T_DAT:     //date
            x = strlen(S_DAT);
            vals[cnt][1] = palloc((x+1) * sizeof(char));
            strncpy(vals[cnt][1], S_DAT, x);
            vals[cnt][1][x] = '\0';
            break;
        case T_WRD:     //word
            x = strlen(S_WRD);
            vals[cnt][1] = palloc((x+1) * sizeof(char));
            strncpy(vals[cnt][1], S_WRD, x);
            vals[cnt][1][x] = '\0';
            break;
        default:
            elog(ERROR,"Unknown flexme type: %d", type);
            break;
    }
    //value
    vals[cnt][0] = palloc((yyleng+1) * sizeof(char));
    strncpy(vals[cnt][0], yytext, yyleng);
    vals[cnt][0][yyleng] = '\0';

    cnt++;
//    elog(NOTICE,"i: %d", cnt);

    return 0;
}

char ***flexme(char *ptr)
{

    YY_BUFFER_STATE bp;
    int   yyerr = 0;
    cnt = 0;

    //initial table size
    vals = palloc(mem_cnt * sizeof(char *));

    bp = yy_scan_string(ptr);
    yy_switch_to_buffer(bp);
    yyerr = yylex();
    yy_delete_buffer(bp);

    if (yyerr != 0) {
        elog(ERROR, "Flex parser error code: %d", yyerr);
    }

    return vals;
}



File 2 (PG function, which includes flex output analyser of compiled
File 1 - lex.yy.c):

#include "postgres.h"
#include "fmgr.h"
#include "funcapi.h"

#include "lex.yy.c"

char *text_to_cstring(const text *t);   //this is copied directly from
PG sources
char *
text_to_cstring(const text *t)
{
        /* must cast away the const, unfortunately */
        text           *tunpacked = pg_detoast_datum_packed((struct
varlena *) t);
        int                        len = VARSIZE_ANY_EXHDR(tunpacked);
        char           *result;

        result = (char *) palloc(len + 1);
        memcpy(result, VARDATA_ANY(tunpacked), len);
        result[len] = '\0';

        if (tunpacked != t)
                pfree(tunpacked);

        return result;
}


PG_FUNCTION_INFO_V1(flex_me);
Datum    flex_me(PG_FUNCTION_ARGS);

Datum
flex_me(PG_FUNCTION_ARGS) {
    text             *in = PG_GETARG_TEXT_P(0);

    FuncCallContext  *funcctx;
    TupleDesc        tupdesc;
    AttInMetadata    *attinmeta;
    int              call_cntr, max_calls;
    char             ***values;
    char             *ptr;

    // stuff done only on the first call of the function
    if (SRF_IS_FIRSTCALL()) {
        MemoryContext oldcontext;

        // create a function context for cross-call persistence
        funcctx = SRF_FIRSTCALL_INIT();

        // switch to memory context appropriate for multiple  function calls
        oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx);

        ptr = text_to_cstring_imm(in);
        values = flexme(ptr);

        //free char pointer
        pfree(ptr);

        // Build a tuple descriptor for our result type
        if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE)
            ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg
               ("function returning record called in context "
                "that cannot accept type record")));

        // generate attribute metadata needed later to produce
        //   tuples from raw C strings
        attinmeta = TupleDescGetAttInMetadata(tupdesc);
        funcctx->attinmeta = attinmeta;

        //pass first list element
        funcctx->user_fctx = values;

        // total number of tuples to be returned
        funcctx->max_calls = cnt;

        //go back to normal memory context
        MemoryContextSwitchTo(oldcontext);
    }

    // stuff done on every call of the function.
    funcctx = SRF_PERCALL_SETUP();
    call_cntr = funcctx->call_cntr;
    max_calls = funcctx->max_calls;
    attinmeta = funcctx->attinmeta;
    values = (char ***) funcctx->user_fctx;

    //set return routine
    if (call_cntr < max_calls) {
        char      **rvals;
        HeapTuple tuple;
        Datum     result;
        int       i;

        // Prepare a values array for building the returned
        //tuple. This should be an array of C strings which
        //will be processed later by the type input functions
        rvals = palloc(2*sizeof(char *));

        //value (text)
        i = strlen(values[call_cntr][0]);
        rvals[0] = palloc((i+1)*sizeof(char));
        strncpy(rvals[0], values[call_cntr][0], i);
        rvals[0][i] = '\0';

        //type (text)
        i = strlen(values[call_cntr][1]);
        rvals[1] = palloc((i+1)*sizeof(char));
        strncpy(rvals[1], values[call_cntr][1], i);
        rvals[1][i] = '\0';

        // build a tuple and make into datum.
        tuple = BuildTupleFromCStrings(attinmeta, rvals);

        result = HeapTupleGetDatum(tuple);


        //free memory
        pfree(rvals[0]);
        pfree(rvals[1]);
        pfree(rvals);
        pfree(values[call_cntr][0]);
        pfree(values[call_cntr][1]);
        pfree(values[call_cntr]);

        //return datum
        SRF_RETURN_NEXT(funcctx, result);
    }
    else {
        SRF_RETURN_DONE(funcctx);
    }

    return true;
}