* relation. Then it is a simple matter to emit the output demanded by the
* SQL spec for INTERSECT, INTERSECT ALL, EXCEPT, or EXCEPT ALL.
*
- * In SETOP_HASHED mode, the input is delivered in no particular order.
- * We build a hash table in memory with one entry for each group of
- * identical tuples, and count the number of tuples in the group from
- * each relation. After seeing all the input, we scan the hashtable and
- * generate the correct output using those counts.
+ * In SETOP_HASHED mode, the input is delivered in no particular order,
+ * except that we know all the tuples from one input relation will come before
+ * all the tuples of the other. The planner guarantees that the first input
+ * relation is the left-hand one for EXCEPT, and tries to make the smaller
+ * input relation come first for INTERSECT. We build a hash table in memory
+ * with one entry for each group of identical tuples, and count the number of
+ * tuples in the group from each relation. After seeing all the input, we
+ * scan the hashtable and generate the correct output using those counts.
+ * We can avoid making hashtable entries for any tuples appearing only in the
+ * second input relation, since they cannot result in any output.
*
* This node type is not used for UNION or UNION ALL, since those can be
* implemented more cheaply (there's no need for the junk attribute to
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/executor/nodeSetOp.c,v 1.26 2008/08/07 03:04:03 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/executor/nodeSetOp.c,v 1.27 2008/08/07 19:35:02 tgl Exp $
*
*-------------------------------------------------------------------------
*/
/*
* Initialize state for a new group of input values.
*/
-static void
-initialize_counts(SetOpState *setopstate, SetOpStatePerGroup pergroup)
+static inline void
+initialize_counts(SetOpStatePerGroup pergroup)
{
pergroup->numLeft = pergroup->numRight = 0;
}
/*
* Advance the appropriate counter for one input tuple.
*/
-static void
-advance_counts(SetOpState *setopstate, SetOpStatePerGroup pergroup,
- TupleTableSlot *inputslot)
+static inline void
+advance_counts(SetOpStatePerGroup pergroup, int flag)
+{
+ if (flag)
+ pergroup->numRight++;
+ else
+ pergroup->numLeft++;
+}
+
+/*
+ * Fetch the "flag" column from an input tuple.
+ * This is an integer column with value 0 for left side, 1 for right side.
+ */
+static int
+fetch_tuple_flag(SetOpState *setopstate, TupleTableSlot *inputslot)
{
SetOp *node = (SetOp *) setopstate->ps.plan;
int flag;
node->flagColIdx,
&isNull));
Assert(!isNull);
- if (flag)
- pergroup->numRight++;
- else
- pergroup->numLeft++;
+ Assert(flag == 0 || flag == 1);
+ return flag;
}
/*
setopstate->tempContext);
}
-/*
- * Find or create a hashtable entry for the tuple group containing the
- * given tuple.
- */
-static SetOpHashEntry
-lookup_hash_entry(SetOpState *setopstate, TupleTableSlot *inputslot)
-{
- SetOpHashEntry entry;
- bool isnew;
-
- /* find or create the hashtable entry */
- entry = (SetOpHashEntry) LookupTupleHashEntry(setopstate->hashtable,
- inputslot,
- &isnew);
-
- if (isnew)
- {
- /* initialize counts for new tuple group */
- initialize_counts(setopstate, &entry->pergroup);
- }
-
- /* Must reset temp context after each hashtable lookup */
- MemoryContextReset(setopstate->tempContext);
-
- return entry;
-}
-
/*
* We've completed processing a tuple group. Decide how many copies (if any)
* of its representative row to emit, and store the count into numOutput.
setopstate->grp_firstTuple = NULL; /* don't keep two pointers */
/* Initialize working state for a new input tuple group */
- initialize_counts(setopstate, pergroup);
+ initialize_counts(pergroup);
/* Count the first input tuple */
- advance_counts(setopstate, pergroup, resultTupleSlot);
+ advance_counts(pergroup,
+ fetch_tuple_flag(setopstate, resultTupleSlot));
/*
* Scan the outer plan until we exhaust it or cross a group boundary.
}
/* Still in same group, so count this tuple */
- advance_counts(setopstate, pergroup, outerslot);
+ advance_counts(pergroup,
+ fetch_tuple_flag(setopstate, outerslot));
}
/*
static void
setop_fill_hash_table(SetOpState *setopstate)
{
+ SetOp *node = (SetOp *) setopstate->ps.plan;
PlanState *outerPlan;
- SetOpHashEntry entry;
- TupleTableSlot *outerslot;
+ int firstFlag;
+ bool in_first_rel;
/*
* get state info from node
*/
outerPlan = outerPlanState(setopstate);
+ firstFlag = node->firstFlag;
+ /* verify planner didn't mess up */
+ Assert(firstFlag == 0 ||
+ (firstFlag == 1 &&
+ (node->cmd == SETOPCMD_INTERSECT ||
+ node->cmd == SETOPCMD_INTERSECT_ALL)));
/*
* Process each outer-plan tuple, and then fetch the next one, until we
* exhaust the outer plan.
*/
+ in_first_rel = true;
for (;;)
{
+ TupleTableSlot *outerslot;
+ int flag;
+ SetOpHashEntry entry;
+ bool isnew;
+
outerslot = ExecProcNode(outerPlan);
if (TupIsNull(outerslot))
break;
- /* Find or build hashtable entry for this tuple's group */
- entry = lookup_hash_entry(setopstate, outerslot);
+ /* Identify whether it's left or right input */
+ flag = fetch_tuple_flag(setopstate, outerslot);
+
+ if (flag == firstFlag)
+ {
+ /* (still) in first input relation */
+ Assert(in_first_rel);
+
+ /* Find or build hashtable entry for this tuple's group */
+ entry = (SetOpHashEntry)
+ LookupTupleHashEntry(setopstate->hashtable, outerslot, &isnew);
+
+ /* If new tuple group, initialize counts */
+ if (isnew)
+ initialize_counts(&entry->pergroup);
+
+ /* Advance the counts */
+ advance_counts(&entry->pergroup, flag);
+ }
+ else
+ {
+ /* reached second relation */
+ in_first_rel = false;
+
+ /* For tuples not seen previously, do not make hashtable entry */
+ entry = (SetOpHashEntry)
+ LookupTupleHashEntry(setopstate->hashtable, outerslot, NULL);
+
+ /* Advance the counts if entry is already present */
+ if (entry)
+ advance_counts(&entry->pergroup, flag);
+ }
- /* Advance the counts */
- advance_counts(setopstate, &entry->pergroup, outerslot);
+ /* Must reset temp context after each hashtable lookup */
+ MemoryContextReset(setopstate->tempContext);
}
setopstate->table_filled = true;
*
*
* IDENTIFICATION
- * $PostgreSQL: pgsql/src/backend/optimizer/prep/prepunion.c,v 1.151 2008/08/07 03:04:03 tgl Exp $
+ * $PostgreSQL: pgsql/src/backend/optimizer/prep/prepunion.c,v 1.152 2008/08/07 19:35:02 tgl Exp $
*
*-------------------------------------------------------------------------
*/
#include "parser/parsetree.h"
#include "utils/lsyscache.h"
#include "utils/rel.h"
+#include "utils/selfuncs.h"
static Plan *recurse_set_operations(Node *setOp, PlannerInfo *root,
double tuple_fraction,
List *colTypes, bool junkOK,
int flag, List *refnames_tlist,
- List **sortClauses);
+ List **sortClauses, double *pNumGroups);
static Plan *generate_union_plan(SetOperationStmt *op, PlannerInfo *root,
double tuple_fraction,
- List *refnames_tlist, List **sortClauses);
+ List *refnames_tlist,
+ List **sortClauses, double *pNumGroups);
static Plan *generate_nonunion_plan(SetOperationStmt *op, PlannerInfo *root,
double tuple_fraction,
- List *refnames_tlist, List **sortClauses);
+ List *refnames_tlist,
+ List **sortClauses, double *pNumGroups);
static List *recurse_union_children(Node *setOp, PlannerInfo *root,
double tuple_fraction,
SetOperationStmt *top_union,
List **sortClauses);
static bool choose_hashed_setop(PlannerInfo *root, List *groupClauses,
Plan *input_plan,
- double tuple_fraction, double dNumDistinctRows,
+ double dNumGroups, double dNumOutputRows,
+ double tuple_fraction,
const char *construct);
static List *generate_setop_tlist(List *colTypes, int flag,
Index varno,
return recurse_set_operations((Node *) topop, root, tuple_fraction,
topop->colTypes, true, -1,
leftmostQuery->targetList,
- sortClauses);
+ sortClauses, NULL);
}
/*
* junkOK: if true, child resjunk columns may be left in the result
* flag: if >= 0, add a resjunk output column indicating value of flag
* refnames_tlist: targetlist to take column names from
+ *
+ * Returns a plan for the subtree, as well as these output parameters:
* *sortClauses: receives list of SortGroupClauses for result plan, if any
+ * *pNumGroups: if not NULL, we estimate the number of distinct groups
+ * in the result, and store it there
*
* We don't have to care about typmods here: the only allowed difference
* between set-op input and output typmods is input is a specific typmod
double tuple_fraction,
List *colTypes, bool junkOK,
int flag, List *refnames_tlist,
- List **sortClauses)
+ List **sortClauses, double *pNumGroups)
{
if (IsA(setOp, RangeTblRef))
{
tuple_fraction,
&subroot);
+ /*
+ * Estimate number of groups if caller wants it. If the subquery
+ * used grouping or aggregation, its output is probably mostly
+ * unique anyway; otherwise do statistical estimation.
+ */
+ if (pNumGroups)
+ {
+ if (subquery->groupClause || subquery->distinctClause ||
+ subroot->hasHavingQual || subquery->hasAggs)
+ *pNumGroups = subplan->plan_rows;
+ else
+ *pNumGroups = estimate_num_groups(subroot,
+ get_tlist_exprs(subquery->targetList, false),
+ subplan->plan_rows);
+ }
+
/*
* Add a SubqueryScan with the caller-requested targetlist
*/
if (op->op == SETOP_UNION)
plan = generate_union_plan(op, root, tuple_fraction,
refnames_tlist,
- sortClauses);
+ sortClauses, pNumGroups);
else
plan = generate_nonunion_plan(op, root, tuple_fraction,
refnames_tlist,
- sortClauses);
+ sortClauses, pNumGroups);
/*
* If necessary, add a Result node to project the caller-requested
generate_union_plan(SetOperationStmt *op, PlannerInfo *root,
double tuple_fraction,
List *refnames_tlist,
- List **sortClauses)
+ List **sortClauses, double *pNumGroups)
{
List *planlist;
List *tlist;
else
plan = make_union_unique(op, plan, root, tuple_fraction, sortClauses);
+ /*
+ * Estimate number of groups if caller wants it. For now we just
+ * assume the output is unique --- this is certainly true for the
+ * UNION case, and we want worst-case estimates anyway.
+ */
+ if (pNumGroups)
+ *pNumGroups = plan->plan_rows;
+
return plan;
}
generate_nonunion_plan(SetOperationStmt *op, PlannerInfo *root,
double tuple_fraction,
List *refnames_tlist,
- List **sortClauses)
+ List **sortClauses, double *pNumGroups)
{
Plan *lplan,
*rplan,
*groupList,
*planlist,
*child_sortclauses;
- double dNumDistinctRows;
- double dNumOutputRows;
- long numDistinctRows;
+ double dLeftGroups,
+ dRightGroups,
+ dNumGroups,
+ dNumOutputRows;
+ long numGroups;
bool use_hash;
SetOpCmd cmd;
+ int firstFlag;
/* Recurse on children, ensuring their outputs are marked */
lplan = recurse_set_operations(op->larg, root,
0.0 /* all tuples needed */ ,
op->colTypes, false, 0,
refnames_tlist,
- &child_sortclauses);
+ &child_sortclauses, &dLeftGroups);
rplan = recurse_set_operations(op->rarg, root,
0.0 /* all tuples needed */ ,
op->colTypes, false, 1,
refnames_tlist,
- &child_sortclauses);
- planlist = list_make2(lplan, rplan);
+ &child_sortclauses, &dRightGroups);
+
+ /*
+ * For EXCEPT, we must put the left input first. For INTERSECT, either
+ * order should give the same results, and we prefer to put the smaller
+ * input first in order to minimize the size of the hash table in the
+ * hashing case. "Smaller" means the one with the fewer groups.
+ */
+ if (op->op == SETOP_EXCEPT || dLeftGroups <= dRightGroups)
+ {
+ planlist = list_make2(lplan, rplan);
+ firstFlag = 0;
+ }
+ else
+ {
+ planlist = list_make2(rplan, lplan);
+ firstFlag = 1;
+ }
/*
* Generate tlist for Append plan node.
}
/*
- * XXX for the moment, take the number of distinct groups as being the
- * total input size, ie, the worst case. This is too conservative, but
- * we don't want to risk having the hashtable overrun memory; also,
- * it's not clear how to get a decent estimate of the true size.
+ * Estimate number of distinct groups that we'll need hashtable entries
+ * for; this is the size of the left-hand input for EXCEPT, or the smaller
+ * input for INTERSECT. Also estimate the number of eventual output rows.
+ * In non-ALL cases, we estimate each group produces one output row;
+ * in ALL cases use the relevant relation size. These are worst-case
+ * estimates, of course, but we need to be conservative.
*/
- dNumDistinctRows = plan->plan_rows;
+ if (op->op == SETOP_EXCEPT)
+ {
+ dNumGroups = dLeftGroups;
+ dNumOutputRows = op->all ? lplan->plan_rows : dNumGroups;
+ }
+ else
+ {
+ dNumGroups = Min(dLeftGroups, dRightGroups);
+ dNumOutputRows = op->all ? Min(lplan->plan_rows, rplan->plan_rows) : dNumGroups;
+ }
/* Also convert to long int --- but 'ware overflow! */
- numDistinctRows = (long) Min(dNumDistinctRows, (double) LONG_MAX);
-
- /*
- * The output size is taken as 10% of that, which is a completely bogus
- * guess, but it's what we've used historically.
- */
- dNumOutputRows = ceil(dNumDistinctRows * 0.1);
+ numGroups = (long) Min(dNumGroups, (double) LONG_MAX);
/*
* Decide whether to hash or sort, and add a sort node if needed.
*/
use_hash = choose_hashed_setop(root, groupList, plan,
- tuple_fraction, dNumDistinctRows,
+ dNumGroups, dNumOutputRows, tuple_fraction,
(op->op == SETOP_INTERSECT) ? "INTERSECT" : "EXCEPT");
if (!use_hash)
break;
}
plan = (Plan *) make_setop(cmd, use_hash ? SETOP_HASHED : SETOP_SORTED,
- plan, groupList, list_length(op->colTypes) + 1,
- numDistinctRows, dNumOutputRows);
+ plan, groupList,
+ list_length(op->colTypes) + 1,
+ use_hash ? firstFlag : -1,
+ numGroups, dNumOutputRows);
/* Result is sorted only if we're not hashing */
*sortClauses = use_hash ? NIL : groupList;
+ if (pNumGroups)
+ *pNumGroups = dNumGroups;
+
return plan;
}
tuple_fraction,
top_union->colTypes, false,
-1, refnames_tlist,
- &child_sortclauses));
+ &child_sortclauses, NULL));
}
/*
List **sortClauses)
{
List *groupList;
- double dNumDistinctRows;
- long numDistinctRows;
+ double dNumGroups;
+ long numGroups;
/* Identify the grouping semantics */
groupList = generate_setop_grouplist(op, plan->targetlist);
}
/*
- * XXX for the moment, take the number of distinct groups as being the
- * total input size, ie, the worst case. This is too conservative, but
- * we don't want to risk having the hashtable overrun memory; also,
+ * XXX for the moment, take the number of distinct groups as equal to
+ * the total input size, ie, the worst case. This is too conservative,
+ * but we don't want to risk having the hashtable overrun memory; also,
* it's not clear how to get a decent estimate of the true size. One
* should note as well the propensity of novices to write UNION rather
* than UNION ALL even when they don't expect any duplicates...
*/
- dNumDistinctRows = plan->plan_rows;
+ dNumGroups = plan->plan_rows;
/* Also convert to long int --- but 'ware overflow! */
- numDistinctRows = (long) Min(dNumDistinctRows, (double) LONG_MAX);
+ numGroups = (long) Min(dNumGroups, (double) LONG_MAX);
/* Decide whether to hash or sort */
if (choose_hashed_setop(root, groupList, plan,
- tuple_fraction, dNumDistinctRows,
+ dNumGroups, dNumGroups, tuple_fraction,
"UNION"))
{
/* Hashed aggregate plan --- no sort needed */
extract_grouping_cols(groupList,
plan->targetlist),
extract_grouping_ops(groupList),
- numDistinctRows,
+ numGroups,
0,
plan);
/* Hashed aggregation produces randomly-ordered results */
/* Sort and Unique */
plan = (Plan *) make_sort_from_sortclauses(root, groupList, plan);
plan = (Plan *) make_unique(plan, groupList);
- plan->plan_rows = dNumDistinctRows;
+ plan->plan_rows = dNumGroups;
/* We know the sort order of the result */
*sortClauses = groupList;
}
static bool
choose_hashed_setop(PlannerInfo *root, List *groupClauses,
Plan *input_plan,
- double tuple_fraction, double dNumDistinctRows,
+ double dNumGroups, double dNumOutputRows,
+ double tuple_fraction,
const char *construct)
{
- int numDistinctCols = list_length(groupClauses);
+ int numGroupCols = list_length(groupClauses);
bool can_sort;
bool can_hash;
Size hashentrysize;
- List *needed_pathkeys;
Path hashed_p;
Path sorted_p;
*/
hashentrysize = MAXALIGN(input_plan->plan_width) + MAXALIGN(sizeof(MinimalTupleData));
- if (hashentrysize * dNumDistinctRows > work_mem * 1024L)
+ if (hashentrysize * dNumGroups > work_mem * 1024L)
return false;
/*
* make actual Paths for these steps.
*/
cost_agg(&hashed_p, root, AGG_HASHED, 0,
- numDistinctCols, dNumDistinctRows,
+ numGroupCols, dNumGroups,
input_plan->startup_cost, input_plan->total_cost,
input_plan->plan_rows);
*/
sorted_p.startup_cost = input_plan->startup_cost;
sorted_p.total_cost = input_plan->total_cost;
- /* XXX this is more expensive than cost_sort really needs: */
- needed_pathkeys = make_pathkeys_for_sortclauses(root,
- groupClauses,
- input_plan->targetlist,
- true);
- cost_sort(&sorted_p, root, needed_pathkeys, sorted_p.total_cost,
+ /* XXX cost_sort doesn't actually look at pathkeys, so just pass NIL */
+ cost_sort(&sorted_p, root, NIL, sorted_p.total_cost,
input_plan->plan_rows, input_plan->plan_width, -1.0);
- cost_group(&sorted_p, root, numDistinctCols, dNumDistinctRows,
+ cost_group(&sorted_p, root, numGroupCols, dNumGroups,
sorted_p.startup_cost, sorted_p.total_cost,
input_plan->plan_rows);
* have to convert an absolute count (LIMIT) into fractional form.
*/
if (tuple_fraction >= 1.0)
- tuple_fraction /= dNumDistinctRows;
+ tuple_fraction /= dNumOutputRows;
if (compare_fractional_path_costs(&hashed_p, &sorted_p,
tuple_fraction) < 0)
projects / postgresql / commitdiff