1 /*-------------------------------------------------------------------------
4 * foreign-data wrapper for server-side flat files.
6 * Copyright (c) 2010-2014, PostgreSQL Global Development Group
9 * contrib/file_fdw/file_fdw.c
11 *-------------------------------------------------------------------------
18 #include "access/htup_details.h"
19 #include "access/reloptions.h"
20 #include "access/sysattr.h"
21 #include "catalog/pg_foreign_table.h"
22 #include "commands/copy.h"
23 #include "commands/defrem.h"
24 #include "commands/explain.h"
25 #include "commands/vacuum.h"
26 #include "foreign/fdwapi.h"
27 #include "foreign/foreign.h"
28 #include "miscadmin.h"
29 #include "nodes/makefuncs.h"
30 #include "optimizer/cost.h"
31 #include "optimizer/pathnode.h"
32 #include "optimizer/planmain.h"
33 #include "optimizer/restrictinfo.h"
34 #include "optimizer/var.h"
35 #include "utils/memutils.h"
36 #include "utils/rel.h"
41 * Describes the valid options for objects that use this wrapper.
46 Oid optcontext; /* Oid of catalog in which option may appear */
50 * Valid options for file_fdw.
51 * These options are based on the options for the COPY FROM command.
52 * But note that force_not_null and force_null are handled as boolean options
53 * attached to a column, not as table options.
55 * Note: If you are adding new option for user mapping, you need to modify
56 * fileGetOptions(), which currently doesn't bother to look at user mappings.
58 static const struct FileFdwOption valid_options[] = {
60 {"filename", ForeignTableRelationId},
63 /* oids option is not supported */
64 {"format", ForeignTableRelationId},
65 {"header", ForeignTableRelationId},
66 {"delimiter", ForeignTableRelationId},
67 {"quote", ForeignTableRelationId},
68 {"escape", ForeignTableRelationId},
69 {"null", ForeignTableRelationId},
70 {"encoding", ForeignTableRelationId},
71 {"force_not_null", AttributeRelationId},
72 {"force_null", AttributeRelationId},
74 * force_quote is not supported by file_fdw because it's for COPY TO.
82 * FDW-specific information for RelOptInfo.fdw_private.
84 typedef struct FileFdwPlanState
86 char *filename; /* file to read */
87 List *options; /* merged COPY options, excluding filename */
88 BlockNumber pages; /* estimate of file's physical size */
89 double ntuples; /* estimate of number of rows in file */
93 * FDW-specific information for ForeignScanState.fdw_state.
95 typedef struct FileFdwExecutionState
97 char *filename; /* file to read */
98 List *options; /* merged COPY options, excluding filename */
99 CopyState cstate; /* state of reading file */
100 } FileFdwExecutionState;
105 PG_FUNCTION_INFO_V1(file_fdw_handler);
106 PG_FUNCTION_INFO_V1(file_fdw_validator);
109 * FDW callback routines
111 static void fileGetForeignRelSize(PlannerInfo *root,
114 static void fileGetForeignPaths(PlannerInfo *root,
117 static ForeignScan *fileGetForeignPlan(PlannerInfo *root,
120 ForeignPath *best_path,
123 static void fileExplainForeignScan(ForeignScanState *node, ExplainState *es);
124 static void fileBeginForeignScan(ForeignScanState *node, int eflags);
125 static TupleTableSlot *fileIterateForeignScan(ForeignScanState *node);
126 static void fileReScanForeignScan(ForeignScanState *node);
127 static void fileEndForeignScan(ForeignScanState *node);
128 static bool fileAnalyzeForeignTable(Relation relation,
129 AcquireSampleRowsFunc *func,
130 BlockNumber *totalpages);
135 static bool is_valid_option(const char *option, Oid context);
136 static void fileGetOptions(Oid foreigntableid,
137 char **filename, List **other_options);
138 static List *get_file_fdw_attribute_options(Oid relid);
139 static bool check_selective_binary_conversion(RelOptInfo *baserel,
142 static void estimate_size(PlannerInfo *root, RelOptInfo *baserel,
143 FileFdwPlanState *fdw_private);
144 static void estimate_costs(PlannerInfo *root, RelOptInfo *baserel,
145 FileFdwPlanState *fdw_private,
146 Cost *startup_cost, Cost *total_cost);
147 static int file_acquire_sample_rows(Relation onerel, int elevel,
148 HeapTuple *rows, int targrows,
149 double *totalrows, double *totaldeadrows);
153 * Foreign-data wrapper handler function: return a struct with pointers
154 * to my callback routines.
157 file_fdw_handler(PG_FUNCTION_ARGS)
159 FdwRoutine *fdwroutine = makeNode(FdwRoutine);
161 fdwroutine->GetForeignRelSize = fileGetForeignRelSize;
162 fdwroutine->GetForeignPaths = fileGetForeignPaths;
163 fdwroutine->GetForeignPlan = fileGetForeignPlan;
164 fdwroutine->ExplainForeignScan = fileExplainForeignScan;
165 fdwroutine->BeginForeignScan = fileBeginForeignScan;
166 fdwroutine->IterateForeignScan = fileIterateForeignScan;
167 fdwroutine->ReScanForeignScan = fileReScanForeignScan;
168 fdwroutine->EndForeignScan = fileEndForeignScan;
169 fdwroutine->AnalyzeForeignTable = fileAnalyzeForeignTable;
171 PG_RETURN_POINTER(fdwroutine);
175 * Validate the generic options given to a FOREIGN DATA WRAPPER, SERVER,
176 * USER MAPPING or FOREIGN TABLE that uses file_fdw.
178 * Raise an ERROR if the option or its value is considered invalid.
181 file_fdw_validator(PG_FUNCTION_ARGS)
183 List *options_list = untransformRelOptions(PG_GETARG_DATUM(0));
184 Oid catalog = PG_GETARG_OID(1);
185 char *filename = NULL;
186 DefElem *force_not_null = NULL;
187 DefElem *force_null = NULL;
188 List *other_options = NIL;
192 * Only superusers are allowed to set options of a file_fdw foreign table.
193 * This is because the filename is one of those options, and we don't want
194 * non-superusers to be able to determine which file gets read.
196 * Putting this sort of permissions check in a validator is a bit of a
197 * crock, but there doesn't seem to be any other place that can enforce
198 * the check more cleanly.
200 * Note that the valid_options[] array disallows setting filename at any
201 * options level other than foreign table --- otherwise there'd still be a
204 if (catalog == ForeignTableRelationId && !superuser())
206 (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE),
207 errmsg("only superuser can change options of a file_fdw foreign table")));
210 * Check that only options supported by file_fdw, and allowed for the
211 * current object type, are given.
213 foreach(cell, options_list)
215 DefElem *def = (DefElem *) lfirst(cell);
217 if (!is_valid_option(def->defname, catalog))
219 const struct FileFdwOption *opt;
223 * Unknown option specified, complain about it. Provide a hint
224 * with list of valid options for the object.
226 initStringInfo(&buf);
227 for (opt = valid_options; opt->optname; opt++)
229 if (catalog == opt->optcontext)
230 appendStringInfo(&buf, "%s%s", (buf.len > 0) ? ", " : "",
235 (errcode(ERRCODE_FDW_INVALID_OPTION_NAME),
236 errmsg("invalid option \"%s\"", def->defname),
238 ? errhint("Valid options in this context are: %s",
240 : errhint("There are no valid options in this context.")));
244 * Separate out filename and column-specific options, since
245 * ProcessCopyOptions won't accept them.
248 if (strcmp(def->defname, "filename") == 0)
252 (errcode(ERRCODE_SYNTAX_ERROR),
253 errmsg("conflicting or redundant options")));
254 filename = defGetString(def);
257 * force_not_null is a boolean option; after validation we can discard
258 * it - it will be retrieved later in get_file_fdw_attribute_options()
260 else if (strcmp(def->defname, "force_not_null") == 0)
264 (errcode(ERRCODE_SYNTAX_ERROR),
265 errmsg("conflicting or redundant options"),
266 errhint("option \"force_not_null\" supplied more than once for a column")));
267 force_not_null = def;
268 /* Don't care what the value is, as long as it's a legal boolean */
269 (void) defGetBoolean(def);
271 /* See comments for force_not_null above */
272 else if (strcmp(def->defname, "force_null") == 0)
276 (errcode(ERRCODE_SYNTAX_ERROR),
277 errmsg("conflicting or redundant options"),
278 errhint("option \"force_null\" supplied more than once for a column")));
280 (void) defGetBoolean(def);
283 other_options = lappend(other_options, def);
287 * Now apply the core COPY code's validation logic for more checks.
289 ProcessCopyOptions(NULL, true, other_options);
292 * Filename option is required for file_fdw foreign tables.
294 if (catalog == ForeignTableRelationId && filename == NULL)
296 (errcode(ERRCODE_FDW_DYNAMIC_PARAMETER_VALUE_NEEDED),
297 errmsg("filename is required for file_fdw foreign tables")));
303 * Check if the provided option is one of the valid options.
304 * context is the Oid of the catalog holding the object the option is for.
307 is_valid_option(const char *option, Oid context)
309 const struct FileFdwOption *opt;
311 for (opt = valid_options; opt->optname; opt++)
313 if (context == opt->optcontext && strcmp(opt->optname, option) == 0)
320 * Fetch the options for a file_fdw foreign table.
322 * We have to separate out "filename" from the other options because
323 * it must not appear in the options list passed to the core COPY code.
326 fileGetOptions(Oid foreigntableid,
327 char **filename, List **other_options)
330 ForeignServer *server;
331 ForeignDataWrapper *wrapper;
337 * Extract options from FDW objects. We ignore user mappings because
338 * file_fdw doesn't have any options that can be specified there.
340 * (XXX Actually, given the current contents of valid_options[], there's
341 * no point in examining anything except the foreign table's own options.
344 table = GetForeignTable(foreigntableid);
345 server = GetForeignServer(table->serverid);
346 wrapper = GetForeignDataWrapper(server->fdwid);
349 options = list_concat(options, wrapper->options);
350 options = list_concat(options, server->options);
351 options = list_concat(options, table->options);
352 options = list_concat(options, get_file_fdw_attribute_options(foreigntableid));
355 * Separate out the filename.
361 DefElem *def = (DefElem *) lfirst(lc);
363 if (strcmp(def->defname, "filename") == 0)
365 *filename = defGetString(def);
366 options = list_delete_cell(options, lc, prev);
373 * The validator should have checked that a filename was included in the
374 * options, but check again, just in case.
376 if (*filename == NULL)
377 elog(ERROR, "filename is required for file_fdw foreign tables");
379 *other_options = options;
383 * Retrieve per-column generic options from pg_attribute and construct a list
384 * of DefElems representing them.
386 * At the moment we only have "force_not_null", and "force_null",
387 * which should each be combined into a single DefElem listing all such
388 * columns, since that's what COPY expects.
391 get_file_fdw_attribute_options(Oid relid)
397 List *fnncolumns = NIL;
398 List *fncolumns = NIL;
402 rel = heap_open(relid, AccessShareLock);
403 tupleDesc = RelationGetDescr(rel);
404 natts = tupleDesc->natts;
406 /* Retrieve FDW options for all user-defined attributes. */
407 for (attnum = 1; attnum <= natts; attnum++)
409 Form_pg_attribute attr = tupleDesc->attrs[attnum - 1];
413 /* Skip dropped attributes. */
414 if (attr->attisdropped)
417 options = GetForeignColumnOptions(relid, attnum);
420 DefElem *def = (DefElem *) lfirst(lc);
422 if (strcmp(def->defname, "force_not_null") == 0)
424 if (defGetBoolean(def))
426 char *attname = pstrdup(NameStr(attr->attname));
428 fnncolumns = lappend(fnncolumns, makeString(attname));
431 else if (strcmp(def->defname, "force_null") == 0)
433 if (defGetBoolean(def))
435 char *attname = pstrdup(NameStr(attr->attname));
437 fncolumns = lappend(fncolumns, makeString(attname));
440 /* maybe in future handle other options here */
444 heap_close(rel, AccessShareLock);
446 /* Return DefElem only when some column(s) have force_not_null / force_null options set */
447 if (fnncolumns != NIL)
448 options = lappend(options, makeDefElem("force_not_null", (Node *) fnncolumns));
450 if (fncolumns != NIL)
451 options = lappend(options,makeDefElem("force_null", (Node *) fncolumns));
457 * fileGetForeignRelSize
458 * Obtain relation size estimates for a foreign table
461 fileGetForeignRelSize(PlannerInfo *root,
465 FileFdwPlanState *fdw_private;
468 * Fetch options. We only need filename at this point, but we might as
469 * well get everything and not need to re-fetch it later in planning.
471 fdw_private = (FileFdwPlanState *) palloc(sizeof(FileFdwPlanState));
472 fileGetOptions(foreigntableid,
473 &fdw_private->filename, &fdw_private->options);
474 baserel->fdw_private = (void *) fdw_private;
476 /* Estimate relation size */
477 estimate_size(root, baserel, fdw_private);
481 * fileGetForeignPaths
482 * Create possible access paths for a scan on the foreign table
484 * Currently we don't support any push-down feature, so there is only one
485 * possible access path, which simply returns all records in the order in
489 fileGetForeignPaths(PlannerInfo *root,
493 FileFdwPlanState *fdw_private = (FileFdwPlanState *) baserel->fdw_private;
497 List *coptions = NIL;
499 /* Decide whether to selectively perform binary conversion */
500 if (check_selective_binary_conversion(baserel,
503 coptions = list_make1(makeDefElem("convert_selectively",
507 estimate_costs(root, baserel, fdw_private,
508 &startup_cost, &total_cost);
511 * Create a ForeignPath node and add it as only possible path. We use the
512 * fdw_private list of the path to carry the convert_selectively option;
513 * it will be propagated into the fdw_private list of the Plan node.
515 add_path(baserel, (Path *)
516 create_foreignscan_path(root, baserel,
520 NIL, /* no pathkeys */
521 NULL, /* no outer rel either */
525 * If data file was sorted, and we knew it somehow, we could insert
526 * appropriate pathkeys into the ForeignPath node to tell the planner
533 * Create a ForeignScan plan node for scanning the foreign table
536 fileGetForeignPlan(PlannerInfo *root,
539 ForeignPath *best_path,
543 Index scan_relid = baserel->relid;
546 * We have no native ability to evaluate restriction clauses, so we just
547 * put all the scan_clauses into the plan node's qual list for the
548 * executor to check. So all we have to do here is strip RestrictInfo
549 * nodes from the clauses and ignore pseudoconstants (which will be
550 * handled elsewhere).
552 scan_clauses = extract_actual_clauses(scan_clauses, false);
554 /* Create the ForeignScan node */
555 return make_foreignscan(tlist,
558 NIL, /* no expressions to evaluate */
559 best_path->fdw_private);
563 * fileExplainForeignScan
564 * Produce extra output for EXPLAIN
567 fileExplainForeignScan(ForeignScanState *node, ExplainState *es)
572 /* Fetch options --- we only need filename at this point */
573 fileGetOptions(RelationGetRelid(node->ss.ss_currentRelation),
574 &filename, &options);
576 ExplainPropertyText("Foreign File", filename, es);
578 /* Suppress file size if we're not showing cost details */
581 struct stat stat_buf;
583 if (stat(filename, &stat_buf) == 0)
584 ExplainPropertyLong("Foreign File Size", (long) stat_buf.st_size,
590 * fileBeginForeignScan
591 * Initiate access to the file by creating CopyState
594 fileBeginForeignScan(ForeignScanState *node, int eflags)
596 ForeignScan *plan = (ForeignScan *) node->ss.ps.plan;
600 FileFdwExecutionState *festate;
603 * Do nothing in EXPLAIN (no ANALYZE) case. node->fdw_state stays NULL.
605 if (eflags & EXEC_FLAG_EXPLAIN_ONLY)
608 /* Fetch options of foreign table */
609 fileGetOptions(RelationGetRelid(node->ss.ss_currentRelation),
610 &filename, &options);
612 /* Add any options from the plan (currently only convert_selectively) */
613 options = list_concat(options, plan->fdw_private);
616 * Create CopyState from FDW options. We always acquire all columns, so
617 * as to match the expected ScanTupleSlot signature.
619 cstate = BeginCopyFrom(node->ss.ss_currentRelation,
626 * Save state in node->fdw_state. We must save enough information to call
627 * BeginCopyFrom() again.
629 festate = (FileFdwExecutionState *) palloc(sizeof(FileFdwExecutionState));
630 festate->filename = filename;
631 festate->options = options;
632 festate->cstate = cstate;
634 node->fdw_state = (void *) festate;
638 * fileIterateForeignScan
639 * Read next record from the data file and store it into the
640 * ScanTupleSlot as a virtual tuple
642 static TupleTableSlot *
643 fileIterateForeignScan(ForeignScanState *node)
645 FileFdwExecutionState *festate = (FileFdwExecutionState *) node->fdw_state;
646 TupleTableSlot *slot = node->ss.ss_ScanTupleSlot;
648 ErrorContextCallback errcallback;
650 /* Set up callback to identify error line number. */
651 errcallback.callback = CopyFromErrorCallback;
652 errcallback.arg = (void *) festate->cstate;
653 errcallback.previous = error_context_stack;
654 error_context_stack = &errcallback;
657 * The protocol for loading a virtual tuple into a slot is first
658 * ExecClearTuple, then fill the values/isnull arrays, then
659 * ExecStoreVirtualTuple. If we don't find another row in the file, we
660 * just skip the last step, leaving the slot empty as required.
662 * We can pass ExprContext = NULL because we read all columns from the
663 * file, so no need to evaluate default expressions.
665 * We can also pass tupleOid = NULL because we don't allow oids for
668 ExecClearTuple(slot);
669 found = NextCopyFrom(festate->cstate, NULL,
670 slot->tts_values, slot->tts_isnull,
673 ExecStoreVirtualTuple(slot);
675 /* Remove error callback. */
676 error_context_stack = errcallback.previous;
682 * fileReScanForeignScan
683 * Rescan table, possibly with new parameters
686 fileReScanForeignScan(ForeignScanState *node)
688 FileFdwExecutionState *festate = (FileFdwExecutionState *) node->fdw_state;
690 EndCopyFrom(festate->cstate);
692 festate->cstate = BeginCopyFrom(node->ss.ss_currentRelation,
701 * Finish scanning foreign table and dispose objects used for this scan
704 fileEndForeignScan(ForeignScanState *node)
706 FileFdwExecutionState *festate = (FileFdwExecutionState *) node->fdw_state;
708 /* if festate is NULL, we are in EXPLAIN; nothing to do */
710 EndCopyFrom(festate->cstate);
714 * fileAnalyzeForeignTable
715 * Test whether analyzing this foreign table is supported
718 fileAnalyzeForeignTable(Relation relation,
719 AcquireSampleRowsFunc *func,
720 BlockNumber *totalpages)
724 struct stat stat_buf;
726 /* Fetch options of foreign table */
727 fileGetOptions(RelationGetRelid(relation), &filename, &options);
730 * Get size of the file. (XXX if we fail here, would it be better to just
731 * return false to skip analyzing the table?)
733 if (stat(filename, &stat_buf) < 0)
735 (errcode_for_file_access(),
736 errmsg("could not stat file \"%s\": %m",
740 * Convert size to pages. Must return at least 1 so that we can tell
741 * later on that pg_class.relpages is not default.
743 *totalpages = (stat_buf.st_size + (BLCKSZ - 1)) / BLCKSZ;
747 *func = file_acquire_sample_rows;
753 * check_selective_binary_conversion
755 * Check to see if it's useful to convert only a subset of the file's columns
756 * to binary. If so, construct a list of the column names to be converted,
757 * return that at *columns, and return TRUE. (Note that it's possible to
758 * determine that no columns need be converted, for instance with a COUNT(*)
759 * query. So we can't use returning a NIL list to indicate failure.)
762 check_selective_binary_conversion(RelOptInfo *baserel,
771 Bitmapset *attrs_used = NULL;
772 bool has_wholerow = false;
776 *columns = NIL; /* default result */
779 * Check format of the file. If binary format, this is irrelevant.
781 table = GetForeignTable(foreigntableid);
782 foreach(lc, table->options)
784 DefElem *def = (DefElem *) lfirst(lc);
786 if (strcmp(def->defname, "format") == 0)
788 char *format = defGetString(def);
790 if (strcmp(format, "binary") == 0)
796 /* Collect all the attributes needed for joins or final output. */
797 pull_varattnos((Node *) baserel->reltargetlist, baserel->relid,
800 /* Add all the attributes used by restriction clauses. */
801 foreach(lc, baserel->baserestrictinfo)
803 RestrictInfo *rinfo = (RestrictInfo *) lfirst(lc);
805 pull_varattnos((Node *) rinfo->clause, baserel->relid,
809 /* Convert attribute numbers to column names. */
810 rel = heap_open(foreigntableid, AccessShareLock);
811 tupleDesc = RelationGetDescr(rel);
813 while ((attnum = bms_first_member(attrs_used)) >= 0)
815 /* Adjust for system attributes. */
816 attnum += FirstLowInvalidHeapAttributeNumber;
824 /* Ignore system attributes. */
828 /* Get user attributes. */
831 Form_pg_attribute attr = tupleDesc->attrs[attnum - 1];
832 char *attname = NameStr(attr->attname);
834 /* Skip dropped attributes (probably shouldn't see any here). */
835 if (attr->attisdropped)
837 *columns = lappend(*columns, makeString(pstrdup(attname)));
841 /* Count non-dropped user attributes while we have the tupdesc. */
843 for (i = 0; i < tupleDesc->natts; i++)
845 Form_pg_attribute attr = tupleDesc->attrs[i];
847 if (attr->attisdropped)
852 heap_close(rel, AccessShareLock);
854 /* If there's a whole-row reference, fail: we need all the columns. */
861 /* If all the user attributes are needed, fail. */
862 if (numattrs == list_length(*columns))
872 * Estimate size of a foreign table.
874 * The main result is returned in baserel->rows. We also set
875 * fdw_private->pages and fdw_private->ntuples for later use in the cost
879 estimate_size(PlannerInfo *root, RelOptInfo *baserel,
880 FileFdwPlanState *fdw_private)
882 struct stat stat_buf;
888 * Get size of the file. It might not be there at plan time, though, in
889 * which case we have to use a default estimate.
891 if (stat(fdw_private->filename, &stat_buf) < 0)
892 stat_buf.st_size = 10 * BLCKSZ;
895 * Convert size to pages for use in I/O cost estimate later.
897 pages = (stat_buf.st_size + (BLCKSZ - 1)) / BLCKSZ;
900 fdw_private->pages = pages;
903 * Estimate the number of tuples in the file.
905 if (baserel->pages > 0)
908 * We have # of pages and # of tuples from pg_class (that is, from a
909 * previous ANALYZE), so compute a tuples-per-page estimate and scale
910 * that by the current file size.
914 density = baserel->tuples / (double) baserel->pages;
915 ntuples = clamp_row_est(density * (double) pages);
920 * Otherwise we have to fake it. We back into this estimate using the
921 * planner's idea of the relation width; which is bogus if not all
922 * columns are being read, not to mention that the text representation
923 * of a row probably isn't the same size as its internal
924 * representation. Possibly we could do something better, but the
925 * real answer to anyone who complains is "ANALYZE" ...
929 tuple_width = MAXALIGN(baserel->width) +
930 MAXALIGN(sizeof(HeapTupleHeaderData));
931 ntuples = clamp_row_est((double) stat_buf.st_size /
932 (double) tuple_width);
934 fdw_private->ntuples = ntuples;
937 * Now estimate the number of rows returned by the scan after applying the
938 * baserestrictinfo quals.
941 clauselist_selectivity(root,
942 baserel->baserestrictinfo,
947 nrows = clamp_row_est(nrows);
949 /* Save the output-rows estimate for the planner */
950 baserel->rows = nrows;
954 * Estimate costs of scanning a foreign table.
956 * Results are returned in *startup_cost and *total_cost.
959 estimate_costs(PlannerInfo *root, RelOptInfo *baserel,
960 FileFdwPlanState *fdw_private,
961 Cost *startup_cost, Cost *total_cost)
963 BlockNumber pages = fdw_private->pages;
964 double ntuples = fdw_private->ntuples;
969 * We estimate costs almost the same way as cost_seqscan(), thus assuming
970 * that I/O costs are equivalent to a regular table file of the same size.
971 * However, we take per-tuple CPU costs as 10x of a seqscan, to account
972 * for the cost of parsing records.
974 run_cost += seq_page_cost * pages;
976 *startup_cost = baserel->baserestrictcost.startup;
977 cpu_per_tuple = cpu_tuple_cost * 10 + baserel->baserestrictcost.per_tuple;
978 run_cost += cpu_per_tuple * ntuples;
979 *total_cost = *startup_cost + run_cost;
983 * file_acquire_sample_rows -- acquire a random sample of rows from the table
985 * Selected rows are returned in the caller-allocated array rows[],
986 * which must have at least targrows entries.
987 * The actual number of rows selected is returned as the function result.
988 * We also count the total number of rows in the file and return it into
989 * *totalrows. Note that *totaldeadrows is always set to 0.
991 * Note that the returned list of rows is not always in order by physical
992 * position in the file. Therefore, correlation estimates derived later
993 * may be meaningless, but it's OK because we don't use the estimates
994 * currently (the planner only pays attention to correlation for indexscans).
997 file_acquire_sample_rows(Relation onerel, int elevel,
998 HeapTuple *rows, int targrows,
999 double *totalrows, double *totaldeadrows)
1002 double rowstoskip = -1; /* -1 means not set yet */
1011 ErrorContextCallback errcallback;
1012 MemoryContext oldcontext = CurrentMemoryContext;
1013 MemoryContext tupcontext;
1016 Assert(targrows > 0);
1018 tupDesc = RelationGetDescr(onerel);
1019 values = (Datum *) palloc(tupDesc->natts * sizeof(Datum));
1020 nulls = (bool *) palloc(tupDesc->natts * sizeof(bool));
1022 /* Fetch options of foreign table */
1023 fileGetOptions(RelationGetRelid(onerel), &filename, &options);
1026 * Create CopyState from FDW options.
1028 cstate = BeginCopyFrom(onerel, filename, false, NIL, options);
1031 * Use per-tuple memory context to prevent leak of memory used to read
1032 * rows from the file with Copy routines.
1034 tupcontext = AllocSetContextCreate(CurrentMemoryContext,
1035 "file_fdw temporary context",
1036 ALLOCSET_DEFAULT_MINSIZE,
1037 ALLOCSET_DEFAULT_INITSIZE,
1038 ALLOCSET_DEFAULT_MAXSIZE);
1040 /* Prepare for sampling rows */
1041 rstate = anl_init_selection_state(targrows);
1043 /* Set up callback to identify error line number. */
1044 errcallback.callback = CopyFromErrorCallback;
1045 errcallback.arg = (void *) cstate;
1046 errcallback.previous = error_context_stack;
1047 error_context_stack = &errcallback;
1053 /* Check for user-requested abort or sleep */
1054 vacuum_delay_point();
1056 /* Fetch next row */
1057 MemoryContextReset(tupcontext);
1058 MemoryContextSwitchTo(tupcontext);
1060 found = NextCopyFrom(cstate, NULL, values, nulls, NULL);
1062 MemoryContextSwitchTo(oldcontext);
1068 * The first targrows sample rows are simply copied into the
1069 * reservoir. Then we start replacing tuples in the sample until we
1070 * reach the end of the relation. This algorithm is from Jeff Vitter's
1071 * paper (see more info in commands/analyze.c).
1073 if (numrows < targrows)
1075 rows[numrows++] = heap_form_tuple(tupDesc, values, nulls);
1080 * t in Vitter's paper is the number of records already processed.
1081 * If we need to compute a new S value, we must use the
1082 * not-yet-incremented value of totalrows as t.
1085 rowstoskip = anl_get_next_S(*totalrows, targrows, &rstate);
1087 if (rowstoskip <= 0)
1090 * Found a suitable tuple, so save it, replacing one old tuple
1093 int k = (int) (targrows * anl_random_fract());
1095 Assert(k >= 0 && k < targrows);
1096 heap_freetuple(rows[k]);
1097 rows[k] = heap_form_tuple(tupDesc, values, nulls);
1106 /* Remove error callback. */
1107 error_context_stack = errcallback.previous;
1110 MemoryContextDelete(tupcontext);
1112 EndCopyFrom(cstate);
1118 * Emit some interesting relation info
1121 (errmsg("\"%s\": file contains %.0f rows; "
1122 "%d rows in sample",
1123 RelationGetRelationName(onerel),
1124 *totalrows, numrows)));