1 /*-------------------------------------------------------------------------
4 * Routines to handle hash join nodes
6 * Portions Copyright (c) 1996-2003, PostgreSQL Global Development Group
7 * Portions Copyright (c) 1994, Regents of the University of California
11 * $Header: /cvsroot/pgsql/src/backend/executor/nodeHashjoin.c,v 1.57 2003/09/25 06:57:59 petere Exp $
13 *-------------------------------------------------------------------------
18 #include "executor/executor.h"
19 #include "executor/nodeHash.h"
20 #include "executor/nodeHashjoin.h"
21 #include "optimizer/clauses.h"
22 #include "utils/memutils.h"
25 static TupleTableSlot *ExecHashJoinOuterGetTuple(PlanState *node,
26 HashJoinState *hjstate);
27 static TupleTableSlot *ExecHashJoinGetSavedTuple(HashJoinState *hjstate,
29 TupleTableSlot *tupleSlot);
30 static int ExecHashJoinNewBatch(HashJoinState *hjstate);
33 /* ----------------------------------------------------------------
36 * This function implements the Hybrid Hashjoin algorithm.
37 * recursive partitioning remains to be added.
38 * Note: the relation we build hash table on is the inner
39 * the other one is outer.
40 * ----------------------------------------------------------------
42 TupleTableSlot * /* return: a tuple or NULL */
43 ExecHashJoin(HashJoinState *node)
53 TupleTableSlot *inntuple;
54 ExprContext *econtext;
56 HashJoinTable hashtable;
58 TupleTableSlot *outerTupleSlot;
62 * get information from HashJoin node
64 hjclauses = node->hashclauses;
65 estate = node->js.ps.state;
66 joinqual = node->js.joinqual;
67 otherqual = node->js.ps.qual;
68 hashNode = (HashState *) innerPlanState(node);
69 outerNode = outerPlanState(node);
70 dir = estate->es_direction;
73 * get information from HashJoin state
75 hashtable = node->hj_HashTable;
76 outerkeys = node->hj_OuterHashKeys;
77 econtext = node->js.ps.ps_ExprContext;
80 * Check to see if we're still projecting out tuples from a previous
81 * join tuple (because there is a function-returning-set in the
82 * projection expressions). If so, try to project another one.
84 if (node->js.ps.ps_TupFromTlist)
86 TupleTableSlot *result;
88 result = ExecProject(node->js.ps.ps_ProjInfo, &isDone);
89 if (isDone == ExprMultipleResult)
91 /* Done with that source tuple... */
92 node->js.ps.ps_TupFromTlist = false;
96 * If we're doing an IN join, we want to return at most one row per
97 * outer tuple; so we can stop scanning the inner scan if we matched
98 * on the previous try.
100 if (node->js.jointype == JOIN_IN &&
101 node->hj_MatchedOuter)
102 node->hj_NeedNewOuter = true;
105 * Reset per-tuple memory context to free any expression evaluation
106 * storage allocated in the previous tuple cycle. Note this can't
107 * happen until we're done projecting out tuples from a join tuple.
109 ResetExprContext(econtext);
112 * if this is the first call, build the hash table for inner relation
114 if (!node->hj_hashdone)
117 * create the hash table
119 Assert(hashtable == NULL);
120 hashtable = ExecHashTableCreate((Hash *) hashNode->ps.plan,
121 node->hj_HashOperators);
122 node->hj_HashTable = hashtable;
125 * execute the Hash node, to build the hash table
127 hashNode->hashtable = hashtable;
128 (void) ExecProcNode((PlanState *) hashNode);
131 * Open temp files for outer batches, if needed. Note that file
132 * buffers are palloc'd in regular executor context.
134 for (i = 0; i < hashtable->nbatch; i++)
135 hashtable->outerBatchFile[i] = BufFileCreateTemp(false);
137 node->hj_hashdone = true;
141 * Now get an outer tuple and probe into the hash table for matches
143 outerTupleSlot = node->js.ps.ps_OuterTupleSlot;
148 * If we don't have an outer tuple, get the next one
150 if (node->hj_NeedNewOuter)
152 outerTupleSlot = ExecHashJoinOuterGetTuple(outerNode,
154 if (TupIsNull(outerTupleSlot))
160 node->js.ps.ps_OuterTupleSlot = outerTupleSlot;
161 econtext->ecxt_outertuple = outerTupleSlot;
162 node->hj_NeedNewOuter = false;
163 node->hj_MatchedOuter = false;
166 * now we have an outer tuple, find the corresponding bucket
167 * for this tuple from the hash table
169 node->hj_CurBucketNo = ExecHashGetBucket(hashtable, econtext,
171 node->hj_CurTuple = NULL;
174 * Now we've got an outer tuple and the corresponding hash
175 * bucket, but this tuple may not belong to the current batch.
176 * This need only be checked in the first pass.
178 if (hashtable->curbatch == 0)
180 int batchno = ExecHashGetBatch(node->hj_CurBucketNo,
186 * Need to postpone this outer tuple to a later batch.
187 * Save it in the corresponding outer-batch file.
189 hashtable->outerBatchSize[batchno]++;
190 ExecHashJoinSaveTuple(outerTupleSlot->val,
191 hashtable->outerBatchFile[batchno]);
192 node->hj_NeedNewOuter = true;
193 continue; /* loop around for a new outer tuple */
199 * OK, scan the selected hash bucket for matches
203 curtuple = ExecScanHashBucket(node,
206 if (curtuple == NULL)
207 break; /* out of matches */
210 * we've got a match, but still need to test non-hashed quals
212 inntuple = ExecStoreTuple(curtuple,
213 node->hj_HashTupleSlot,
215 false); /* don't pfree this tuple */
216 econtext->ecxt_innertuple = inntuple;
218 /* reset temp memory each time to avoid leaks from qual expr */
219 ResetExprContext(econtext);
222 * if we pass the qual, then save state for next call and have
223 * ExecProject form the projection, store it in the tuple
224 * table, and return the slot.
226 * Only the joinquals determine MatchedOuter status, but all
227 * quals must pass to actually return the tuple.
229 if (ExecQual(joinqual, econtext, false))
231 node->hj_MatchedOuter = true;
233 if (otherqual == NIL || ExecQual(otherqual, econtext, false))
235 TupleTableSlot *result;
237 result = ExecProject(node->js.ps.ps_ProjInfo, &isDone);
239 if (isDone != ExprEndResult)
241 node->js.ps.ps_TupFromTlist =
242 (isDone == ExprMultipleResult);
248 * If we didn't return a tuple, may need to set
251 if (node->js.jointype == JOIN_IN)
253 node->hj_NeedNewOuter = true;
254 break; /* out of loop over hash bucket */
260 * Now the current outer tuple has run out of matches, so check
261 * whether to emit a dummy outer-join tuple. If not, loop around
262 * to get a new outer tuple.
264 node->hj_NeedNewOuter = true;
266 if (!node->hj_MatchedOuter &&
267 node->js.jointype == JOIN_LEFT)
270 * We are doing an outer join and there were no join matches
271 * for this outer tuple. Generate a fake join tuple with
272 * nulls for the inner tuple, and return it if it passes the
275 econtext->ecxt_innertuple = node->hj_NullInnerTupleSlot;
277 if (ExecQual(otherqual, econtext, false))
280 * qualification was satisfied so we project and return
281 * the slot containing the result tuple using
284 TupleTableSlot *result;
286 result = ExecProject(node->js.ps.ps_ProjInfo, &isDone);
288 if (isDone != ExprEndResult)
290 node->js.ps.ps_TupFromTlist =
291 (isDone == ExprMultipleResult);
299 /* ----------------------------------------------------------------
302 * Init routine for HashJoin node.
303 * ----------------------------------------------------------------
306 ExecInitHashJoin(HashJoin *node, EState *estate)
308 HashJoinState *hjstate;
316 * create state structure
318 hjstate = makeNode(HashJoinState);
319 hjstate->js.ps.plan = (Plan *) node;
320 hjstate->js.ps.state = estate;
323 * Miscellaneous initialization
325 * create expression context for node
327 ExecAssignExprContext(estate, &hjstate->js.ps);
330 * initialize child expressions
332 hjstate->js.ps.targetlist = (List *)
333 ExecInitExpr((Expr *) node->join.plan.targetlist,
334 (PlanState *) hjstate);
335 hjstate->js.ps.qual = (List *)
336 ExecInitExpr((Expr *) node->join.plan.qual,
337 (PlanState *) hjstate);
338 hjstate->js.jointype = node->join.jointype;
339 hjstate->js.joinqual = (List *)
340 ExecInitExpr((Expr *) node->join.joinqual,
341 (PlanState *) hjstate);
342 hjstate->hashclauses = (List *)
343 ExecInitExpr((Expr *) node->hashclauses,
344 (PlanState *) hjstate);
347 * initialize child nodes
349 outerNode = outerPlan(node);
350 hashNode = (Hash *) innerPlan(node);
352 outerPlanState(hjstate) = ExecInitNode(outerNode, estate);
353 innerPlanState(hjstate) = ExecInitNode((Plan *) hashNode, estate);
355 #define HASHJOIN_NSLOTS 3
358 * tuple table initialization
360 ExecInitResultTupleSlot(estate, &hjstate->js.ps);
361 hjstate->hj_OuterTupleSlot = ExecInitExtraTupleSlot(estate);
363 switch (node->join.jointype)
369 hjstate->hj_NullInnerTupleSlot =
370 ExecInitNullTupleSlot(estate,
371 ExecGetResultType(innerPlanState(hjstate)));
374 elog(ERROR, "unrecognized join type: %d",
375 (int) node->join.jointype);
379 * now for some voodoo. our temporary tuple slot is actually the
380 * result tuple slot of the Hash node (which is our inner plan). we
381 * do this because Hash nodes don't return tuples via ExecProcNode()
382 * -- instead the hash join node uses ExecScanHashBucket() to get at
383 * the contents of the hash table. -cim 6/9/91
386 HashState *hashstate = (HashState *) innerPlanState(hjstate);
387 TupleTableSlot *slot = hashstate->ps.ps_ResultTupleSlot;
389 hjstate->hj_HashTupleSlot = slot;
393 * initialize tuple type and projection info
395 ExecAssignResultTypeFromTL(&hjstate->js.ps);
396 ExecAssignProjectionInfo(&hjstate->js.ps);
398 ExecSetSlotDescriptor(hjstate->hj_OuterTupleSlot,
399 ExecGetResultType(outerPlanState(hjstate)),
403 * initialize hash-specific info
406 hjstate->hj_hashdone = false;
407 hjstate->hj_HashTable = (HashJoinTable) NULL;
409 hjstate->hj_CurBucketNo = 0;
410 hjstate->hj_CurTuple = (HashJoinTuple) NULL;
413 * The planner already made a list of the inner hashkeys for us, but
414 * we also need a list of the outer hashkeys, as well as a list of the
415 * hash operator OIDs. Both lists of exprs must then be prepared for
418 hjstate->hj_InnerHashKeys = (List *)
419 ExecInitExpr((Expr *) hashNode->hashkeys,
420 innerPlanState(hjstate));
421 ((HashState *) innerPlanState(hjstate))->hashkeys =
422 hjstate->hj_InnerHashKeys;
426 foreach(hcl, node->hashclauses)
428 OpExpr *hclause = (OpExpr *) lfirst(hcl);
430 Assert(IsA(hclause, OpExpr));
431 hclauses = lappend(hclauses, get_leftop((Expr *) hclause));
432 hoperators = lappendo(hoperators, hclause->opno);
434 hjstate->hj_OuterHashKeys = (List *)
435 ExecInitExpr((Expr *) hclauses,
436 (PlanState *) hjstate);
437 hjstate->hj_HashOperators = hoperators;
439 hjstate->js.ps.ps_OuterTupleSlot = NULL;
440 hjstate->js.ps.ps_TupFromTlist = false;
441 hjstate->hj_NeedNewOuter = true;
442 hjstate->hj_MatchedOuter = false;
448 ExecCountSlotsHashJoin(HashJoin *node)
450 return ExecCountSlotsNode(outerPlan(node)) +
451 ExecCountSlotsNode(innerPlan(node)) +
455 /* ----------------------------------------------------------------
458 * clean up routine for HashJoin node
459 * ----------------------------------------------------------------
462 ExecEndHashJoin(HashJoinState *node)
467 if (node->hj_HashTable)
469 ExecHashTableDestroy(node->hj_HashTable);
470 node->hj_HashTable = NULL;
474 * Free the exprcontext
476 ExecFreeExprContext(&node->js.ps);
479 * clean out the tuple table
481 ExecClearTuple(node->js.ps.ps_ResultTupleSlot);
482 ExecClearTuple(node->hj_OuterTupleSlot);
483 ExecClearTuple(node->hj_HashTupleSlot);
488 ExecEndNode(outerPlanState(node));
489 ExecEndNode(innerPlanState(node));
492 /* ----------------------------------------------------------------
493 * ExecHashJoinOuterGetTuple
495 * get the next outer tuple for hashjoin: either by
496 * executing a plan node as in the first pass, or from
497 * the tmp files for the hashjoin batches.
498 * ----------------------------------------------------------------
501 static TupleTableSlot *
502 ExecHashJoinOuterGetTuple(PlanState *node, HashJoinState *hjstate)
504 HashJoinTable hashtable = hjstate->hj_HashTable;
505 int curbatch = hashtable->curbatch;
506 TupleTableSlot *slot;
509 { /* if it is the first pass */
510 slot = ExecProcNode(node);
511 if (!TupIsNull(slot))
515 * We have just reached the end of the first pass. Try to switch
518 curbatch = ExecHashJoinNewBatch(hjstate);
522 * Try to read from a temp file. Loop allows us to advance to new
525 while (curbatch <= hashtable->nbatch)
527 slot = ExecHashJoinGetSavedTuple(hjstate,
528 hashtable->outerBatchFile[curbatch - 1],
529 hjstate->hj_OuterTupleSlot);
530 if (!TupIsNull(slot))
532 curbatch = ExecHashJoinNewBatch(hjstate);
535 /* Out of batches... */
539 /* ----------------------------------------------------------------
540 * ExecHashJoinGetSavedTuple
542 * read the next tuple from a tmp file
543 * ----------------------------------------------------------------
546 static TupleTableSlot *
547 ExecHashJoinGetSavedTuple(HashJoinState *hjstate,
549 TupleTableSlot *tupleSlot)
555 nread = BufFileRead(file, (void *) &htup, sizeof(HeapTupleData));
557 return NULL; /* end of file */
558 if (nread != sizeof(HeapTupleData))
560 (errcode_for_file_access(),
561 errmsg("could not read from hash-join temporary file: %m")));
562 heapTuple = palloc(HEAPTUPLESIZE + htup.t_len);
563 memcpy((char *) heapTuple, (char *) &htup, sizeof(HeapTupleData));
564 heapTuple->t_datamcxt = CurrentMemoryContext;
565 heapTuple->t_data = (HeapTupleHeader)
566 ((char *) heapTuple + HEAPTUPLESIZE);
567 nread = BufFileRead(file, (void *) heapTuple->t_data, htup.t_len);
568 if (nread != (size_t) htup.t_len)
570 (errcode_for_file_access(),
571 errmsg("could not read from hash-join temporary file: %m")));
572 return ExecStoreTuple(heapTuple, tupleSlot, InvalidBuffer, true);
575 /* ----------------------------------------------------------------
576 * ExecHashJoinNewBatch
578 * switch to a new hashjoin batch
579 * ----------------------------------------------------------------
582 ExecHashJoinNewBatch(HashJoinState *hjstate)
584 HashJoinTable hashtable = hjstate->hj_HashTable;
585 int nbatch = hashtable->nbatch;
586 int newbatch = hashtable->curbatch + 1;
587 long *innerBatchSize = hashtable->innerBatchSize;
588 long *outerBatchSize = hashtable->outerBatchSize;
590 TupleTableSlot *slot;
591 ExprContext *econtext;
597 * We no longer need the previous outer batch file; close it right
598 * away to free disk space.
600 BufFileClose(hashtable->outerBatchFile[newbatch - 2]);
601 hashtable->outerBatchFile[newbatch - 2] = NULL;
605 * We can skip over any batches that are empty on either side. Release
606 * associated temp files right away.
608 while (newbatch <= nbatch &&
609 (innerBatchSize[newbatch - 1] == 0L ||
610 outerBatchSize[newbatch - 1] == 0L))
612 BufFileClose(hashtable->innerBatchFile[newbatch - 1]);
613 hashtable->innerBatchFile[newbatch - 1] = NULL;
614 BufFileClose(hashtable->outerBatchFile[newbatch - 1]);
615 hashtable->outerBatchFile[newbatch - 1] = NULL;
619 if (newbatch > nbatch)
620 return newbatch; /* no more batches */
623 * Rewind inner and outer batch files for this batch, so that we can
624 * start reading them.
626 if (BufFileSeek(hashtable->outerBatchFile[newbatch - 1], 0, 0L, SEEK_SET))
628 (errcode_for_file_access(),
629 errmsg("could not rewind hash-join temporary file: %m")));
631 innerFile = hashtable->innerBatchFile[newbatch - 1];
633 if (BufFileSeek(innerFile, 0, 0L, SEEK_SET))
635 (errcode_for_file_access(),
636 errmsg("could not rewind hash-join temporary file: %m")));
639 * Reload the hash table with the new inner batch
641 ExecHashTableReset(hashtable, innerBatchSize[newbatch - 1]);
643 econtext = hjstate->js.ps.ps_ExprContext;
644 innerhashkeys = hjstate->hj_InnerHashKeys;
646 while ((slot = ExecHashJoinGetSavedTuple(hjstate,
648 hjstate->hj_HashTupleSlot))
651 econtext->ecxt_innertuple = slot;
652 ExecHashTableInsert(hashtable, econtext, innerhashkeys);
656 * after we build the hash table, the inner batch file is no longer
659 BufFileClose(innerFile);
660 hashtable->innerBatchFile[newbatch - 1] = NULL;
662 hashtable->curbatch = newbatch;
666 /* ----------------------------------------------------------------
667 * ExecHashJoinSaveTuple
669 * save a tuple to a tmp file.
671 * The data recorded in the file for each tuple is an image of its
672 * HeapTupleData (with meaningless t_data pointer) followed by the
673 * HeapTupleHeader and tuple data.
674 * ----------------------------------------------------------------
678 ExecHashJoinSaveTuple(HeapTuple heapTuple,
683 written = BufFileWrite(file, (void *) heapTuple, sizeof(HeapTupleData));
684 if (written != sizeof(HeapTupleData))
686 (errcode_for_file_access(),
687 errmsg("could not write to hash-join temporary file: %m")));
688 written = BufFileWrite(file, (void *) heapTuple->t_data, heapTuple->t_len);
689 if (written != (size_t) heapTuple->t_len)
691 (errcode_for_file_access(),
692 errmsg("could not write to hash-join temporary file: %m")));
696 ExecReScanHashJoin(HashJoinState *node, ExprContext *exprCtxt)
699 * If we haven't yet built the hash table then we can just return;
700 * nothing done yet, so nothing to undo.
702 if (!node->hj_hashdone)
704 Assert(node->hj_HashTable != NULL);
707 * In a multi-batch join, we currently have to do rescans the hard
708 * way, primarily because batch temp files may have already been
709 * released. But if it's a single-batch join, and there is no
710 * parameter change for the inner subnode, then we can just re-use the
711 * existing hash table without rebuilding it.
713 if (node->hj_HashTable->nbatch == 0 &&
714 ((PlanState *) node)->righttree->chgParam == NULL)
716 /* okay to reuse the hash table; needn't rescan inner, either */
720 /* must destroy and rebuild hash table */
721 node->hj_hashdone = false;
722 ExecHashTableDestroy(node->hj_HashTable);
723 node->hj_HashTable = NULL;
726 * if chgParam of subnode is not null then plan will be re-scanned
727 * by first ExecProcNode.
729 if (((PlanState *) node)->righttree->chgParam == NULL)
730 ExecReScan(((PlanState *) node)->righttree, exprCtxt);
733 /* Always reset intra-tuple state */
734 node->hj_CurBucketNo = 0;
735 node->hj_CurTuple = (HashJoinTuple) NULL;
737 node->js.ps.ps_OuterTupleSlot = (TupleTableSlot *) NULL;
738 node->js.ps.ps_TupFromTlist = false;
739 node->hj_NeedNewOuter = true;
740 node->hj_MatchedOuter = false;
743 * if chgParam of subnode is not null then plan will be re-scanned by
744 * first ExecProcNode.
746 if (((PlanState *) node)->lefttree->chgParam == NULL)
747 ExecReScan(((PlanState *) node)->lefttree, exprCtxt);