Re: Determining client_encoding from client locale

Here's my first attempt at setting client_encoding automatically from
locale.

It adds a new conninfo parameter to libpq, "client_encoding". If set to
"auto", libpq uses the encoding as returned by
pg_get_encoding_from_locale(). Any other value is passed through to the
server as is.

psql is modified to set "client_encoding=auto", unless overridden by
PGCLIENTENCODING.


BTW, I had to modify psql to use PQconnectdb() instead of
PQsetdblogin(), so that it can pass the extra parameter. I found it a
bit laboursome to construct the conninfo string with proper escaping,
just to have libpq parse and split it into components again. Could we
have a version of PQconnectdb() with an API more suited for setting the
params programmatically? The PQsetdbLogin() approach doesn't scale as
parameters are added/removed in future versions, but we could have
something like this:

PGconn *PQconnectParams(const char **params)

Where "params" is an array with an even number of parameters, forming
key/value pairs. Usage example:

char *connparams[] = {
    "dbname", "mydb",
    "user", username,
    NULL /* terminate with NULL */
};
conn = PQconnectParams(connparams);

This is similar to what I did internally in psql in the attached patch.

Another idea is to use an array of PQconninfoOption structs:

PQconn *PQconnectParams(PQconninfoOption *params);

This would be quite natural since that's the format returned by
PQconnDefaults() and PQconninfoParse(), but a bit more cumbersome to use
in applications that don't use those functions, as in the previous example.

--
  Heikki Linnakangas
  EnterpriseDB   http://www.enterprisedb.com
commit 24f6d68ddd3725c1f9a98c47f7535b2973ffc492
Author: Heikki Linnakangas <heikki@enterprisedb.com>
Date:   Mon Jul 6 16:54:00 2009 +0300

    Add client_encoding conninfo parameter. By specifying special value
    'auto', libpq will determine the encoding from the current locale.

    Modify psql to use the 'auto' mode if PGCLIENTENCODING if not set.

diff --git a/doc/src/sgml/libpq.sgml b/doc/src/sgml/libpq.sgml
index 86affb0..a5d45b2 100644
--- a/doc/src/sgml/libpq.sgml
+++ b/doc/src/sgml/libpq.sgml
@@ -236,6 +236,19 @@
          </listitem>
         </varlistentry>

+        <varlistentry id="libpq-connect-client-encoding" xreflabel="client_encoding">
+         <term><literal>client_encoding</literal></term>
+         <listitem>
+         <para>
+          Character encoding to use. This sets the <varname>client_encoding</varname>
+          configuration option for this connection. In addition to the values
+          accepted by the corresponding server option, you can use 'auto' to
+          determine the right encoding from the current locale in the client
+          (LC_CTYPE environment variable on Unix systems).
+         </para>
+         </listitem>
+        </varlistentry>
+
         <varlistentry id="libpq-connect-options" xreflabel="options">
          <term><literal>options</literal></term>
          <listitem>
@@ -5871,6 +5884,16 @@ myEventProc(PGEventId evtId, void *evtInfo, void *passThrough)
       linkend="libpq-connect-connect-timeout"> connection parameter.
      </para>
     </listitem>
+
+    <listitem>
+     <para>
+      <indexterm>
+       <primary><envar>PGCLIENTENCODING</envar></primary>
+      </indexterm>
+      <envar>PGCLIENTENCODING</envar> behaves the same as <xref
+      linkend="libpq-connect-client-encoding"> connection parameter.
+     </para>
+    </listitem>
    </itemizedlist>
   </para>

@@ -5907,17 +5930,6 @@ myEventProc(PGEventId evtId, void *evtInfo, void *passThrough)
     <listitem>
      <para>
       <indexterm>
-       <primary><envar>PGCLIENTENCODING</envar></primary>
-      </indexterm>
-      <envar>PGCLIENTENCODING</envar> sets the default client character
-      set encoding.  (Equivalent to <literal>SET client_encoding TO
-      ...</literal>.)
-     </para>
-    </listitem>
-
-    <listitem>
-     <para>
-      <indexterm>
        <primary><envar>PGGEQO</envar></primary>
       </indexterm>
       <envar>PGGEQO</envar> sets the default mode for the genetic query
diff --git a/src/bin/psql/command.c b/src/bin/psql/command.c
index 0955e13..6991e7a 100644
--- a/src/bin/psql/command.c
+++ b/src/bin/psql/command.c
@@ -1239,8 +1239,7 @@ do_connect(char *dbname, char *user, char *host, char *port)

     while (true)
     {
-        n_conn = PQsetdbLogin(host, port, NULL, NULL,
-                              dbname, user, password);
+        n_conn = PSQLconnect(host, port, dbname, user, password);

         /* We can immediately discard the password -- no longer needed */
         if (password)
diff --git a/src/bin/psql/common.c b/src/bin/psql/common.c
index 6b2de37..a5a0b0a 100644
--- a/src/bin/psql/common.c
+++ b/src/bin/psql/common.c
@@ -32,6 +32,8 @@ static bool ExecQueryUsingCursor(const char *query, double *elapsed_msec);
 static bool command_no_begin(const char *query);
 static bool is_select_command(const char *query);

+static char *construct_conninfo(const char * const *optarray);
+
 /*
  * "Safe" wrapper around strdup()
  */
@@ -1538,3 +1540,75 @@ expand_tilde(char **filename)

     return *filename;
 }
+
+/*
+ * Establish a connection. This has an API similar to libpq's PQsetdblogin(),
+ * but we set "client_encoding=auto" if PGCLIENTENCODING environment variable
+ * is not set.
+ */
+PGconn *
+PSQLconnect(const char *host,
+            const char *port,
+            const char *dbname,
+            const char *user,
+            const char *password)
+{
+    char *conninfo;
+    PGconn *ret;
+
+    const char *opts[] = {
+        "host", host,
+        "port", port,
+        "dbname", dbname,
+        "user", user,
+        "password", password,
+        "client_encoding", getenv("PGCLIENTENCODING") ? NULL : "auto",
+        NULL, NULL
+    };
+
+    conninfo = construct_conninfo(opts);
+    ret = PQconnectdb(conninfo);
+    free(conninfo);
+
+    return ret;
+}
+
+
+/*
+ * Given an array of connection option name/value pairs, construct a
+ * conninfo string suitable for PQconnectdb(). The array must be terminated
+ * by a NULL pointer.
+ */
+static char *
+construct_conninfo(const char * const *optarray)
+{
+    PQExpBufferData buf;
+
+    initPQExpBuffer(&buf);
+
+    while(*optarray)
+    {
+        const char *option = optarray[0];
+        const char *value = optarray[1];
+
+        if (value != NULL)
+        {
+            /* write option name */
+            appendPQExpBuffer(&buf, "%s = '", option);
+
+            /* write value, escaping single quotes and backslashes */
+            while(*value)
+            {
+                if (*value == '\'' || *value == '\\')
+                    appendPQExpBufferChar(&buf, '\\');
+                appendPQExpBufferChar(&buf, *(value++));
+            }
+
+            appendPQExpBufferStr(&buf, "' ");
+        }
+
+        optarray+=2;
+    }
+
+    return buf.data;
+}
diff --git a/src/bin/psql/common.h b/src/bin/psql/common.h
index edcb2e5..e962164 100644
--- a/src/bin/psql/common.h
+++ b/src/bin/psql/common.h
@@ -57,6 +57,12 @@ extern PGresult *PSQLexec(const char *query, bool start_xact);

 extern bool SendQuery(const char *query);

+extern PGconn *PSQLconnect(const char *host,
+                           const char *port,
+                           const char *dbname,
+                           const char *user,
+                           const char *password);
+
 extern bool is_superuser(void);
 extern bool standard_strings(void);
 extern const char *session_username(void);
diff --git a/src/bin/psql/startup.c b/src/bin/psql/startup.c
index 429e5d9..cf65a76 100644
--- a/src/bin/psql/startup.c
+++ b/src/bin/psql/startup.c
@@ -172,7 +172,7 @@ main(int argc, char *argv[])
     do
     {
         new_pass = false;
-        pset.db = PQsetdbLogin(options.host, options.port, NULL, NULL,
+        pset.db = PSQLconnect(options.host, options.port,
                     options.action == ACT_LIST_DB && options.dbname == NULL ?
                                "postgres" : options.dbname,
                                options.username, password);
diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c
index 70dfd90..c92fbe6 100644
--- a/src/interfaces/libpq/fe-connect.c
+++ b/src/interfaces/libpq/fe-connect.c
@@ -154,6 +154,9 @@ static const PQconninfoOption PQconninfoOptions[] = {
     {"port", "PGPORT", DEF_PGPORT_STR, NULL,
     "Database-Port", "", 6},

+    {"client_encoding", "PGCLIENTENCODING", NULL, NULL,
+    "Client-Encoding", "", 10},
+
     /*
      * "tty" is no longer used either, but keep it present for backwards
      * compatibility.
@@ -225,9 +228,6 @@ static const PQEnvironmentOption EnvironmentOptions[] =
     {
         "PGTZ", "timezone"
     },
-    {
-        "PGCLIENTENCODING", "client_encoding"
-    },
     /* internal performance-related settings */
     {
         "PGGEQO", "geqo"
@@ -424,6 +424,8 @@ connectOptions1(PGconn *conn, const char *conninfo)
     conn->pgpass = tmp ? strdup(tmp) : NULL;
     tmp = conninfo_getval(connOptions, "connect_timeout");
     conn->connect_timeout = tmp ? strdup(tmp) : NULL;
+    tmp = conninfo_getval(connOptions, "client_encoding");
+    conn->client_encoding_initial = tmp ? strdup(tmp) : NULL;
     tmp = conninfo_getval(connOptions, "sslmode");
     conn->sslmode = tmp ? strdup(tmp) : NULL;
     tmp = conninfo_getval(connOptions, "sslkey");
@@ -552,6 +554,17 @@ connectOptions2(PGconn *conn)
         conn->sslmode = strdup(DefaultSSLMode);

     /*
+     * Resolve 'auto' client_encoding
+     */
+    if (conn->client_encoding_initial &&
+        strcmp(conn->client_encoding_initial, "auto") == 0)
+    {
+        int encid = pg_get_encoding_from_locale(NULL);
+        free(conn->client_encoding_initial);
+        conn->client_encoding_initial = strdup(pg_encoding_to_char(encid));
+    }
+
+    /*
      * Only if we get this far is it appropriate to try to connect. (We need a
      * state flag, rather than just the boolean result of this function, in
      * case someone tries to PQreset() the PGconn.)
@@ -1843,7 +1856,7 @@ keep_going:                        /* We will come back to here until there is
                 if (PG_PROTOCOL_MAJOR(conn->pversion) < 3)
                 {
                     conn->status = CONNECTION_SETENV;
-                    conn->setenv_state = SETENV_STATE_OPTION_SEND;
+                    conn->setenv_state = SETENV_STATE_CLIENT_ENCODING_SEND;
                     conn->next_eo = EnvironmentOptions;
                     return PGRES_POLLING_WRITING;
                 }
@@ -3649,6 +3662,13 @@ PQsetClientEncoding(PGconn *conn, const char *encoding)
     if (!encoding)
         return -1;

+    /* resolve special 'auto' value from the locale */
+    if (strcmp(encoding, "auto") == 0)
+    {
+        int encid = pg_get_encoding_from_locale(NULL);
+        encoding = pg_encoding_to_char(encid);
+    }
+
     /* check query buffer overflow */
     if (sizeof(qbuf) < (sizeof(query) + strlen(encoding)))
         return -1;
diff --git a/src/interfaces/libpq/fe-protocol2.c b/src/interfaces/libpq/fe-protocol2.c
index 87ba65b..8d45706 100644
--- a/src/interfaces/libpq/fe-protocol2.c
+++ b/src/interfaces/libpq/fe-protocol2.c
@@ -58,6 +58,7 @@ pqSetenvPoll(PGconn *conn)
     switch (conn->setenv_state)
     {
             /* These are reading states */
+        case SETENV_STATE_CLIENT_ENCODING_WAIT:
         case SETENV_STATE_OPTION_WAIT:
         case SETENV_STATE_QUERY1_WAIT:
         case SETENV_STATE_QUERY2_WAIT:
@@ -74,6 +75,7 @@ pqSetenvPoll(PGconn *conn)
             }

             /* These are writing states, so we just proceed. */
+        case SETENV_STATE_CLIENT_ENCODING_SEND:
         case SETENV_STATE_OPTION_SEND:
         case SETENV_STATE_QUERY1_SEND:
         case SETENV_STATE_QUERY2_SEND:
@@ -98,6 +100,34 @@ pqSetenvPoll(PGconn *conn)
     {
         switch (conn->setenv_state)
         {
+            case SETENV_STATE_CLIENT_ENCODING_SEND:
+                {
+                    char        setQuery[100];    /* note length limit in
+                                                 * sprintf below */
+                    const char *val = conn->client_encoding_initial;
+
+                    if (val)
+                    {
+                        if (pg_strcasecmp(val, "default") == 0)
+                            sprintf(setQuery, "SET client_encoding = DEFAULT");
+                        else
+                            sprintf(setQuery, "SET client_encoding = '%.60s'",
+                                    val);
+#ifdef CONNECTDEBUG
+                        fprintf(stderr,
+                                "Sending client_encoding with %s\n",
+                                setQuery);
+#endif
+                        if (!PQsendQuery(conn, setQuery))
+                            goto error_return;
+
+                        conn->setenv_state = SETENV_STATE_CLIENT_ENCODING_WAIT;
+                    }
+                    else
+                        conn->setenv_state = SETENV_STATE_OPTION_SEND;
+                    break;
+                }
+
             case SETENV_STATE_OPTION_SEND:
                 {
                     /*
@@ -142,6 +172,31 @@ pqSetenvPoll(PGconn *conn)
                     break;
                 }

+            case SETENV_STATE_CLIENT_ENCODING_WAIT:
+                {
+                    if (PQisBusy(conn))
+                        return PGRES_POLLING_READING;
+
+                    res = PQgetResult(conn);
+
+                    if (res)
+                    {
+                        if (PQresultStatus(res) != PGRES_COMMAND_OK)
+                        {
+                            PQclear(res);
+                            goto error_return;
+                        }
+                        PQclear(res);
+                        /* Keep reading until PQgetResult returns NULL */
+                    }
+                    else
+                    {
+                        /* Query finished, so send the next option */
+                        conn->setenv_state = SETENV_STATE_OPTION_SEND;
+                    }
+                    break;
+                }
+
             case SETENV_STATE_OPTION_WAIT:
                 {
                     if (PQisBusy(conn))
diff --git a/src/interfaces/libpq/fe-protocol3.c b/src/interfaces/libpq/fe-protocol3.c
index 9102b61..a01ace7 100644
--- a/src/interfaces/libpq/fe-protocol3.c
+++ b/src/interfaces/libpq/fe-protocol3.c
@@ -1920,6 +1920,15 @@ build_startup_packet(const PGconn *conn, char *packet,
             strcpy(packet + packet_len, conn->pgoptions);
         packet_len += strlen(conn->pgoptions) + 1;
     }
+    if (conn->client_encoding_initial && conn->client_encoding_initial[0])
+    {
+        if (packet)
+            strcpy(packet + packet_len, "client_encoding");
+        packet_len += strlen("client_encoding") + 1;
+        if (packet)
+            strcpy(packet + packet_len, conn->client_encoding_initial);
+        packet_len += strlen(conn->client_encoding_initial) + 1;
+    }

     /* Add any environment-driven GUC settings needed */
     for (next_eo = options; next_eo->envName; next_eo++)
diff --git a/src/interfaces/libpq/libpq-int.h b/src/interfaces/libpq/libpq-int.h
index ff9e6b8..1d036ba 100644
--- a/src/interfaces/libpq/libpq-int.h
+++ b/src/interfaces/libpq/libpq-int.h
@@ -235,6 +235,8 @@ typedef enum
 /* (this is used only for 2.0-protocol connections) */
 typedef enum
 {
+    SETENV_STATE_CLIENT_ENCODING_SEND,    /* About to send an Environment Option */
+    SETENV_STATE_CLIENT_ENCODING_WAIT,    /* Waiting for above send to complete */
     SETENV_STATE_OPTION_SEND,    /* About to send an Environment Option */
     SETENV_STATE_OPTION_WAIT,    /* Waiting for above send to complete */
     SETENV_STATE_QUERY1_SEND,    /* About to send a status query */
@@ -294,6 +296,7 @@ struct pg_conn
     char       *pgtty;            /* tty on which the backend messages is
                                  * displayed (OBSOLETE, NOT USED) */
     char       *connect_timeout;    /* connection timeout (numeric string) */
+    char       *client_encoding_initial; /* encoding to use */
     char       *pgoptions;        /* options to start the backend with */
     char       *dbName;            /* database name */
     char       *pguser;            /* Postgres username and password, if any */