]> granicus.if.org Git - postgresql/commitdiff
Eliminate query length limitation imposed by pg_client_to_server
authorTom Lane <tgl@sss.pgh.pa.us>
Sat, 11 Sep 1999 22:28:11 +0000 (22:28 +0000)
committerTom Lane <tgl@sss.pgh.pa.us>
Sat, 11 Sep 1999 22:28:11 +0000 (22:28 +0000)
and pg_server_to_client.  Eliminate copy.c's restriction on the length
of a single attribute.

src/backend/commands/copy.c
src/backend/libpq/pqformat.c
src/backend/utils/mb/mbutils.c
src/include/config.h.in

index a25dc0b8a693654a493894d38d8f386ebcb19a7d..35f3d6da0b31470ac617aa0b01fb98a2a353886b 100644 (file)
@@ -6,7 +6,7 @@
  *
  *
  * IDENTIFICATION
- *       $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.86 1999/07/22 02:40:06 tgl Exp $
+ *       $Header: /cvsroot/pgsql/src/backend/commands/copy.c,v 1.87 1999/09/11 22:28:11 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -26,6 +26,7 @@
 #include "commands/copy.h"
 #include "commands/trigger.h"
 #include "executor/executor.h"
+#include "lib/stringinfo.h"
 #include "libpq/libpq.h"
 #include "miscadmin.h"
 #include "utils/acl.h"
@@ -51,14 +52,10 @@ static void GetIndexRelations(Oid main_relation_oid,
                                  int *n_indices,
                                  Relation **index_rels);
 
-#ifdef COPY_PATCH
 static void CopyReadNewline(FILE *fp, int *newline);
 static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline);
-#else
-static char *CopyReadAttribute(FILE *fp, bool *isnull, char *delim);
-#endif
 
-static void CopyAttributeOut(FILE *fp, char *string, char *delim, int is_array);
+static void CopyAttributeOut(FILE *fp, char *string, char *delim);
 static int     CountTuples(Relation relation);
 
 /*
@@ -431,7 +428,7 @@ CopyTo(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
                                {
                                        string = (char *) (*fmgr_faddr(&out_functions[i]))
                                                (value, elements[i], typmod[i]);
-                                       CopyAttributeOut(fp, string, delim, attr[i]->attnelems);
+                                       CopyAttributeOut(fp, string, delim);
                                        pfree(string);
                                }
                                else
@@ -691,18 +688,12 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
        {
                if (!binary)
                {
-#ifdef COPY_PATCH
                        int                     newline = 0;
 
-#endif
                        lineno++;
                        if (oids)
                        {
-#ifdef COPY_PATCH
                                string = CopyReadAttribute(fp, &isnull, delim, &newline);
-#else
-                               string = CopyReadAttribute(fp, &isnull, delim);
-#endif
                                if (string == NULL)
                                        done = 1;
                                else
@@ -710,19 +701,18 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
                                        loaded_oid = oidin(string);
                                        if (loaded_oid < BootstrapObjectIdData)
                                                elog(ERROR, "COPY TEXT: Invalid Oid. line: %d", lineno);
+                                       pfree(string);
                                }
                        }
                        for (i = 0; i < attr_count && !done; i++)
                        {
-#ifdef COPY_PATCH
                                string = CopyReadAttribute(fp, &isnull, delim, &newline);
-#else
-                               string = CopyReadAttribute(fp, &isnull, delim);
-#endif
                                if (isnull)
                                {
                                        values[i] = PointerGetDatum(NULL);
                                        nulls[i] = 'n';
+                                       if (string)
+                                               pfree(string);
                                }
                                else if (string == NULL)
                                        done = 1;
@@ -739,12 +729,11 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
                                        if (!PointerIsValid(values[i]) &&
                                                !(rel->rd_att->attrs[i]->attbyval))
                                                elog(ERROR, "copy from line %d: Bad file format", lineno);
+                                       pfree(string);
                                }
                        }
-#ifdef COPY_PATCH
                        if (!done)
                                CopyReadNewline(fp, &newline);
-#endif
                }
                else
                {                                               /* binary */
@@ -812,11 +801,6 @@ CopyFrom(Relation rel, bool binary, bool oids, FILE *fp, char *delim)
                if (done)
                        continue;
 
-               /*
-                * Does it have any sence ? - vadim 12/14/96
-                *
-                * tupDesc = CreateTupleDesc(attr_count, attr);
-                */
                tuple = heap_formtuple(tupDesc, values, nulls);
                if (oids)
                        tuple->t_data->t_oid = loaded_oid;
@@ -1086,30 +1070,18 @@ GetIndexRelations(Oid main_relation_oid,
        }
 }
 
-#define EXT_ATTLEN     (5 * BLCKSZ)
 
 /*
-   returns 1 is c is in s
+   returns 1 if c is in s
 */
 static bool
 inString(char c, char *s)
 {
-       int                     i;
-
-       if (s)
-       {
-               i = 0;
-               while (s[i] != '\0')
-               {
-                       if (s[i] == c)
-                               return 1;
-                       i++;
-               }
-       }
+       if (s && c)
+               return strchr(s, c) != NULL;
        return 0;
 }
 
-#ifdef COPY_PATCH
 /*
  * Reads input from fp until an end of line is seen.
  */
@@ -1125,64 +1097,57 @@ CopyReadNewline(FILE *fp, int *newline)
        *newline = 0;
 }
 
-#endif
-
 /*
- * Reads input from fp until eof is seen.  If we are reading from standard
- * input, AND we see a dot on a line by itself (a dot followed immediately
- * by a newline), we exit as if we saw eof.  This is so that copy pipelines
- * can be used as standard input.
+ * Read the value of a single attribute.
+ *
+ * Result is either a palloc'd string, or NULL (if EOF or a null attribute).
+ * *isnull is set true if a null attribute, else false.
+ *
+ * delim is the string of acceptable delimiter characters(s).
+ * *newline remembers whether we've seen a newline ending this tuple.
  */
 
 static char *
-#ifdef COPY_PATCH
 CopyReadAttribute(FILE *fp, bool *isnull, char *delim, int *newline)
-#else
-CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
-#endif
 {
-       static char attribute[EXT_ATTLEN];
+       StringInfoData  attribute_buf;
        char            c;
-       int                     done = 0;
-       int                     i = 0;
-
 #ifdef MULTIBYTE
        int                     mblen;
        int                     encoding;
        unsigned char s[2];
+       char       *cvt;
        int                     j;
 
-#endif
-
-#ifdef MULTIBYTE
        encoding = pg_get_client_encoding();
        s[1] = 0;
 #endif
 
-#ifdef COPY_PATCH
        /* if last delimiter was a newline return a NULL attribute */
        if (*newline)
        {
                *isnull = (bool) true;
                return NULL;
        }
-#endif
 
        *isnull = (bool) false;         /* set default */
+
+       initStringInfo(&attribute_buf);
+
        if (CopyGetEof(fp))
-               return NULL;
+               goto endOfFile;
 
-       while (!done)
+       for (;;)
        {
                c = CopyGetChar(fp);
-
                if (CopyGetEof(fp))
-                       return NULL;
-               else if (c == '\\')
+                       goto endOfFile;
+
+               if (c == '\\')
                {
                        c = CopyGetChar(fp);
                        if (CopyGetEof(fp))
-                               return NULL;
+                               goto endOfFile;
                        switch (c)
                        {
                                case '0':
@@ -1212,14 +1177,14 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
                                                        else
                                                        {
                                                                if (CopyGetEof(fp))
-                                                                       return NULL;
+                                                                       goto endOfFile;
                                                                CopyDonePeek(fp, c, 0); /* Return to stream! */
                                                        }
                                                }
                                                else
                                                {
                                                        if (CopyGetEof(fp))
-                                                               return NULL;
+                                                               goto endOfFile;
                                                        CopyDonePeek(fp, c, 0);         /* Return to stream! */
                                                }
                                                c = val & 0377;
@@ -1244,66 +1209,70 @@ CopyReadAttribute(FILE *fp, bool *isnull, char *delim)
                                        c = '\v';
                                        break;
                                case 'N':
-                                       attribute[0] = '\0';            /* just to be safe */
                                        *isnull = (bool) true;
                                        break;
                                case '.':
                                        c = CopyGetChar(fp);
                                        if (c != '\n')
                                                elog(ERROR, "CopyReadAttribute - end of record marker corrupted. line: %d", lineno);
-                                       return NULL;
+                                       goto endOfFile;
                                        break;
                        }
                }
-               else if (inString(c, delim) || c == '\n')
+               else if (c == '\n' || inString(c, delim))
                {
-#ifdef COPY_PATCH
                        if (c == '\n')
                                *newline = 1;
-#endif
-                       done = 1;
+                       break;
                }
-               if (!done)
-                       attribute[i++] = c;
+               appendStringInfoChar(&attribute_buf, c);
 #ifdef MULTIBYTE
+               /* get additional bytes of the char, if any */
                s[0] = c;
                mblen = pg_encoding_mblen(encoding, s);
-               mblen--;
-               for (j = 0; j < mblen; j++)
+               for (j = 1; j < mblen; j++)
                {
                        c = CopyGetChar(fp);
                        if (CopyGetEof(fp))
-                               return NULL;
-                       attribute[i++] = c;
+                               goto endOfFile;
+                       appendStringInfoChar(&attribute_buf, c);
                }
 #endif
-               if (i == EXT_ATTLEN - 1)
-                       elog(ERROR, "CopyReadAttribute - attribute length too long. line: %d", lineno);
        }
-       attribute[i] = '\0';
+
 #ifdef MULTIBYTE
-       return (pg_client_to_server((unsigned char *) attribute, strlen(attribute)));
-#else
-       return &attribute[0];
+       cvt = (char *) pg_client_to_server((unsigned char *) attribute_buf.data,
+                                                                          attribute_buf.len);
+       if (cvt != attribute_buf.data)
+       {
+               pfree(attribute_buf.data);
+               return cvt;
+       }
 #endif
+       return attribute_buf.data;
+
+endOfFile:
+       pfree(attribute_buf.data);
+       return NULL;
 }
 
 static void
-CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
+CopyAttributeOut(FILE *fp, char *server_string, char *delim)
 {
        char       *string;
        char            c;
-
 #ifdef MULTIBYTE
-       int                     mblen;
+       char       *string_start;
        int                     encoding;
+       int                     mblen;
        int                     i;
-
 #endif
 
 #ifdef MULTIBYTE
-       string = pg_server_to_client(server_string, strlen(server_string));
        encoding = pg_get_client_encoding();
+       string = (char *) pg_server_to_client((unsigned char *) server_string,
+                                                                                 strlen(server_string));
+       string_start = string;
 #else
        string = server_string;
 #endif
@@ -1315,33 +1284,20 @@ CopyAttributeOut(FILE *fp, char *server_string, char *delim, int is_array)
        for (; (c = *string) != '\0'; string++)
 #endif
        {
-               if (c == delim[0] || c == '\n' ||
-                       (c == '\\' && !is_array))
+               if (c == delim[0] || c == '\n' || c == '\\')
                        CopySendChar('\\', fp);
-               else if (c == '\\' && is_array)
-               {
-                       if (*(string + 1) == '\\')
-                       {
-                               /* translate \\ to \\\\ */
-                               CopySendChar('\\', fp);
-                               CopySendChar('\\', fp);
-                               CopySendChar('\\', fp);
-                               string++;
-                       }
-                       else if (*(string + 1) == '"')
-                       {
-                               /* translate \" to \\\" */
-                               CopySendChar('\\', fp);
-                               CopySendChar('\\', fp);
-                       }
-               }
 #ifdef MULTIBYTE
                for (i = 0; i < mblen; i++)
                        CopySendChar(*(string + i), fp);
 #else
-               CopySendChar(*string, fp);
+               CopySendChar(c, fp);
 #endif
        }
+
+#ifdef MULTIBYTE
+       if (string_start != server_string)
+               pfree(string_start);    /* pfree pg_server_to_client result */
+#endif
 }
 
 /*
index 1cc715b92a352c5dbf9f79eca23aef79bdf6c015..f6de4af17107937d67493b13a1b581f8301a6764 100644 (file)
@@ -15,7 +15,7 @@
  *
  * Copyright (c) 1994, Regents of the University of California
  *
- *     $Id: pqformat.c,v 1.8 1999/08/31 04:26:37 tgl Exp $
+ *     $Id: pqformat.c,v 1.9 1999/09/11 22:28:05 tgl Exp $
  *
  *-------------------------------------------------------------------------
  */
@@ -125,13 +125,16 @@ void
 pq_sendcountedtext(StringInfo buf, const char *str, int slen)
 {
 #ifdef MULTIBYTE
-       const char *p;
+       char       *p;
 
-       p = (const char *) pg_server_to_client((unsigned char *) str, slen);
+       p = (char *) pg_server_to_client((unsigned char *) str, slen);
        if (p != str)                           /* actual conversion has been done? */
        {
-               str = p;
-               slen = strlen(str);
+               slen = strlen(p);
+               pq_sendint(buf, slen + 4, 4);
+               appendBinaryStringInfo(buf, p, slen);
+               pfree(p);
+               return;
        }
 #endif
        pq_sendint(buf, slen + 4, 4);
@@ -149,15 +152,16 @@ void
 pq_sendstring(StringInfo buf, const char *str)
 {
        int                     slen = strlen(str);
-
 #ifdef MULTIBYTE
-       const char *p;
+       char       *p;
 
-       p = (const char *) pg_server_to_client((unsigned char *) str, slen);
+       p = (char *) pg_server_to_client((unsigned char *) str, slen);
        if (p != str)                           /* actual conversion has been done? */
        {
-               str = p;
-               slen = strlen(str);
+               slen = strlen(p);
+               appendBinaryStringInfo(buf, p, slen + 1);
+               pfree(p);
+               return;
        }
 #endif
        appendBinaryStringInfo(buf, str, slen + 1);
@@ -229,15 +233,15 @@ int
 pq_puttextmessage(char msgtype, const char *str)
 {
        int                     slen = strlen(str);
-
 #ifdef MULTIBYTE
-       const char *p;
+       char       *p;
 
-       p = (const char *) pg_server_to_client((unsigned char *) str, slen);
+       p = (char *) pg_server_to_client((unsigned char *) str, slen);
        if (p != str)                           /* actual conversion has been done? */
        {
-               str = p;
-               slen = strlen(str);
+               int result = pq_putmessage(msgtype, p, strlen(p) + 1);
+               pfree(p);
+               return result;
        }
 #endif
        return pq_putmessage(msgtype, str, slen + 1);
@@ -299,12 +303,12 @@ pq_getint(int *result, int b)
 int
 pq_getstr(StringInfo s)
 {
-       int                     c;
+       int                     result;
 #ifdef MULTIBYTE
        char       *p;
 #endif
 
-       c = pq_getstring(s);
+       result = pq_getstring(s);
 
 #ifdef MULTIBYTE
        p = (char *) pg_client_to_server((unsigned char *) s->data, s->len);
@@ -314,8 +318,9 @@ pq_getstr(StringInfo s)
                s->len = 0;
                s->data[0] = '\0';
                appendBinaryStringInfo(s, p, strlen(p));
+               pfree(p);
        }
 #endif
 
-       return c;
+       return result;
 }
index a63cd37332d23e598841c57ce753bcf20c77bd72..ba22ec347ff720ab677097be3fddb982402a140a 100644 (file)
@@ -3,7 +3,7 @@
  * client encoding and server internal encoding.
  * (currently mule internal code (mic) is used)
  * Tatsuo Ishii
- * $Id: mbutils.c,v 1.8 1999/07/17 20:18:10 momjian Exp $ */
+ * $Id: mbutils.c,v 1.9 1999/09/11 22:28:00 tgl Exp $ */
 
 
 #include "postgres.h"
@@ -93,59 +93,81 @@ pg_get_client_encoding()
 }
 
 /*
- * convert client encoding to server encoding. if server_encoding ==
- * client_encoding or no conversion function exists,
- * returns s. So be careful.
+ * convert client encoding to server encoding.
+ *
+ * CASE 1: if no conversion is required, then the given pointer s is returned.
+ *
+ * CASE 2: if conversion is required, a palloc'd string is returned.
+ *
+ * Callers must check whether return value differs from passed value
+ * to determine whether to pfree the result or not!
+ *
+ * Note: we assume that conversion cannot cause more than a 4-to-1 growth
+ * in the length of the string --- is this enough?
  */
 unsigned char *
 pg_client_to_server(unsigned char *s, int len)
 {
-       static unsigned char b1[MAX_PARSE_BUFFER * 4];          /* is this enough? */
-       static unsigned char b2[MAX_PARSE_BUFFER * 4];          /* is this enough? */
-       unsigned char *p = s;
+       unsigned char *result = s;
+       unsigned char *buf;
 
        if (client_encoding == GetDatabaseEncoding())
-               return (p);
+               return result;
        if (client_to_mic)
        {
-               (*client_to_mic) (s, b1, len);
-               len = strlen(b1);
-               p = b1;
+               buf = (unsigned char *) palloc(len * 4 + 1);
+               (*client_to_mic) (result, buf, len);
+               result = buf;
+               len = strlen(result);
        }
        if (server_from_mic)
        {
-               (*server_from_mic) (p, b2, len);
-               p = b2;
+               buf = (unsigned char *) palloc(len * 4 + 1);
+               (*server_from_mic) (result, buf, len);
+               if (result != s)
+                       pfree(result);          /* release first buffer */
+               result = buf;
        }
-       return (p);
+       return result;
 }
 
 /*
- * convert server encoding to client encoding. if server_encoding ==
- * client_encoding or no conversion function exists,
- * returns s. So be careful.
+ * convert server encoding to client encoding.
+ *
+ * CASE 1: if no conversion is required, then the given pointer s is returned.
+ *
+ * CASE 2: if conversion is required, a palloc'd string is returned.
+ *
+ * Callers must check whether return value differs from passed value
+ * to determine whether to pfree the result or not!
+ *
+ * Note: we assume that conversion cannot cause more than a 4-to-1 growth
+ * in the length of the string --- is this enough?
  */
 unsigned char *
 pg_server_to_client(unsigned char *s, int len)
 {
-       static unsigned char b1[MAX_PARSE_BUFFER * 4];          /* is this enough? */
-       static unsigned char b2[MAX_PARSE_BUFFER * 4];          /* is this enough? */
-       unsigned char *p = s;
+       unsigned char *result = s;
+       unsigned char *buf;
 
        if (client_encoding == GetDatabaseEncoding())
-               return (p);
+               return result;
        if (server_to_mic)
        {
-               (*server_to_mic) (s, b1, len);
-               len = strlen(b1);
-               p = b1;
+               buf = (unsigned char *) palloc(len * 4 + 1);
+               (*server_to_mic) (result, buf, len);
+               result = buf;
+               len = strlen(result);
        }
        if (client_from_mic)
        {
-               (*client_from_mic) (p, b2, len);
-               p = b2;
+               buf = (unsigned char *) palloc(len * 4 + 1);
+               (*client_from_mic) (result, buf, len);
+               if (result != s)
+                       pfree(result);          /* release first buffer */
+               result = buf;
        }
-       return (p);
+       return result;
 }
 
 /* convert a multi-byte string to a wchar */
index e7ae8b302b66475b11bbcdffcc4ece3811f3aa1e..e0711fb1cb559e4c2945eeb10615e88e59a48dcc 100644 (file)
  */
 /* #define TCL_ARRAYS */
 
-/*
- * The following flag allows copying tables from files with number of columns
- * different than the number of attributes setting missing attributes to NULL
- * and ignoring extra columns.  This also avoids the shift of the attributes
- * of the rest of the file if one line has a wrong column count.
- */
-#define COPY_PATCH
-
 /*
  * User locks are handled totally on the application side as long term
  * cooperative locks which extend beyond the normal transaction boundaries.