granicus.if.org Git - postgresql/blob - src/backend/jit/llvm/llvmjit_expr.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * llvmjit_expr.c
   4  *        JIT compile expressions.
   5  *
   6  * Portions Copyright (c) 1996-2019, PostgreSQL Global Development Group
   7  * Portions Copyright (c) 1994, Regents of the University of California
   8  *
   9  *
  10  * IDENTIFICATION
  11  *        src/backend/jit/llvm/llvmjit_expr.c
  12  *
  13  *-------------------------------------------------------------------------
  14  */
  15
  16 #include "postgres.h"
  17
  18 #include <llvm-c/Core.h>
  19 #include <llvm-c/Target.h>
  20
  21 #include "access/htup_details.h"
  22 #include "access/nbtree.h"
  23 #include "access/tupconvert.h"
  24 #include "catalog/objectaccess.h"
  25 #include "catalog/pg_type.h"
  26 #include "executor/execdebug.h"
  27 #include "executor/nodeAgg.h"
  28 #include "executor/nodeSubplan.h"
  29 #include "executor/execExpr.h"
  30 #include "funcapi.h"
  31 #include "jit/llvmjit.h"
  32 #include "jit/llvmjit_emit.h"
  33 #include "miscadmin.h"
  34 #include "nodes/makefuncs.h"
  35 #include "nodes/nodeFuncs.h"
  36 #include "optimizer/planner.h"
  37 #include "parser/parse_coerce.h"
  38 #include "parser/parsetree.h"
  39 #include "pgstat.h"
  40 #include "utils/acl.h"
  41 #include "utils/builtins.h"
  42 #include "utils/date.h"
  43 #include "utils/fmgrtab.h"
  44 #include "utils/lsyscache.h"
  45 #include "utils/memutils.h"
  46 #include "utils/timestamp.h"
  47 #include "utils/typcache.h"
  48 #include "utils/xml.h"
  49
  50
  51 typedef struct CompiledExprState
  52 {
  53         LLVMJitContext *context;
  54         const char *funcname;
  55 } CompiledExprState;
  56
  57
  58 static Datum ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull);
  59
  60 static LLVMValueRef BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b,
  61                         LLVMModuleRef mod, FunctionCallInfo fcinfo,
  62                         LLVMValueRef *v_fcinfo_isnull);
  63 static void build_EvalXFunc(LLVMBuilderRef b, LLVMModuleRef mod,
  64                                 const char *funcname,
  65                                 LLVMValueRef v_state, LLVMValueRef v_econtext,
  66                                 ExprEvalStep *op);
  67 static LLVMValueRef create_LifetimeEnd(LLVMModuleRef mod);
  68
  69
  70 /*
  71  * JIT compile expression.
  72  */
  73 bool
  74 llvm_compile_expr(ExprState *state)
  75 {
  76         PlanState  *parent = state->parent;
  77         int                     i;
  78         char       *funcname;
  79
  80         LLVMJitContext *context = NULL;
  81
  82         LLVMBuilderRef b;
  83         LLVMModuleRef mod;
  84         LLVMTypeRef eval_sig;
  85         LLVMValueRef eval_fn;
  86         LLVMBasicBlockRef entry;
  87         LLVMBasicBlockRef *opblocks;
  88
  89         /* state itself */
  90         LLVMValueRef v_state;
  91         LLVMValueRef v_econtext;
  92
  93         /* returnvalue */
  94         LLVMValueRef v_isnullp;
  95
  96         /* tmp vars in state */
  97         LLVMValueRef v_tmpvaluep;
  98         LLVMValueRef v_tmpisnullp;
  99
 100         /* slots */
 101         LLVMValueRef v_innerslot;
 102         LLVMValueRef v_outerslot;
 103         LLVMValueRef v_scanslot;
 104         LLVMValueRef v_resultslot;
 105
 106         /* nulls/values of slots */
 107         LLVMValueRef v_innervalues;
 108         LLVMValueRef v_innernulls;
 109         LLVMValueRef v_outervalues;
 110         LLVMValueRef v_outernulls;
 111         LLVMValueRef v_scanvalues;
 112         LLVMValueRef v_scannulls;
 113         LLVMValueRef v_resultvalues;
 114         LLVMValueRef v_resultnulls;
 115
 116         /* stuff in econtext */
 117         LLVMValueRef v_aggvalues;
 118         LLVMValueRef v_aggnulls;
 119
 120         instr_time      starttime;
 121         instr_time      endtime;
 122
 123         llvm_enter_fatal_on_oom();
 124
 125         /* get or create JIT context */
 126         if (parent && parent->state->es_jit)
 127         {
 128                 context = (LLVMJitContext *) parent->state->es_jit;
 129         }
 130         else
 131         {
 132                 context = llvm_create_context(parent->state->es_jit_flags);
 133
 134                 if (parent)
 135                 {
 136                         parent->state->es_jit = &context->base;
 137                 }
 138
 139         }
 140
 141         INSTR_TIME_SET_CURRENT(starttime);
 142
 143         mod = llvm_mutable_module(context);
 144
 145         b = LLVMCreateBuilder();
 146
 147         funcname = llvm_expand_funcname(context, "evalexpr");
 148
 149         /* Create the signature and function */
 150         {
 151                 LLVMTypeRef param_types[3];
 152
 153                 param_types[0] = l_ptr(StructExprState);        /* state */
 154                 param_types[1] = l_ptr(StructExprContext);      /* econtext */
 155                 param_types[2] = l_ptr(TypeParamBool);  /* isnull */
 156
 157                 eval_sig = LLVMFunctionType(TypeSizeT,
 158                                                                         param_types, lengthof(param_types),
 159                                                                         false);
 160         }
 161         eval_fn = LLVMAddFunction(mod, funcname, eval_sig);
 162         LLVMSetLinkage(eval_fn, LLVMExternalLinkage);
 163         LLVMSetVisibility(eval_fn, LLVMDefaultVisibility);
 164         llvm_copy_attributes(AttributeTemplate, eval_fn);
 165
 166         entry = LLVMAppendBasicBlock(eval_fn, "entry");
 167
 168         /* build state */
 169         v_state = LLVMGetParam(eval_fn, 0);
 170         v_econtext = LLVMGetParam(eval_fn, 1);
 171         v_isnullp = LLVMGetParam(eval_fn, 2);
 172
 173         LLVMPositionBuilderAtEnd(b, entry);
 174
 175         v_tmpvaluep = LLVMBuildStructGEP(b, v_state,
 176                                                                          FIELDNO_EXPRSTATE_RESVALUE,
 177                                                                          "v.state.resvalue");
 178         v_tmpisnullp = LLVMBuildStructGEP(b, v_state,
 179                                                                           FIELDNO_EXPRSTATE_RESNULL,
 180                                                                           "v.state.resnull");
 181
 182         /* build global slots */
 183         v_scanslot = l_load_struct_gep(b, v_econtext,
 184                                                                    FIELDNO_EXPRCONTEXT_SCANTUPLE,
 185                                                                    "v_scanslot");
 186         v_innerslot = l_load_struct_gep(b, v_econtext,
 187                                                                         FIELDNO_EXPRCONTEXT_INNERTUPLE,
 188                                                                         "v_innerslot");
 189         v_outerslot = l_load_struct_gep(b, v_econtext,
 190                                                                         FIELDNO_EXPRCONTEXT_OUTERTUPLE,
 191                                                                         "v_outerslot");
 192         v_resultslot = l_load_struct_gep(b, v_state,
 193                                                                          FIELDNO_EXPRSTATE_RESULTSLOT,
 194                                                                          "v_resultslot");
 195
 196         /* build global values/isnull pointers */
 197         v_scanvalues = l_load_struct_gep(b, v_scanslot,
 198                                                                          FIELDNO_TUPLETABLESLOT_VALUES,
 199                                                                          "v_scanvalues");
 200         v_scannulls = l_load_struct_gep(b, v_scanslot,
 201                                                                         FIELDNO_TUPLETABLESLOT_ISNULL,
 202                                                                         "v_scannulls");
 203         v_innervalues = l_load_struct_gep(b, v_innerslot,
 204                                                                           FIELDNO_TUPLETABLESLOT_VALUES,
 205                                                                           "v_innervalues");
 206         v_innernulls = l_load_struct_gep(b, v_innerslot,
 207                                                                          FIELDNO_TUPLETABLESLOT_ISNULL,
 208                                                                          "v_innernulls");
 209         v_outervalues = l_load_struct_gep(b, v_outerslot,
 210                                                                           FIELDNO_TUPLETABLESLOT_VALUES,
 211                                                                           "v_outervalues");
 212         v_outernulls = l_load_struct_gep(b, v_outerslot,
 213                                                                          FIELDNO_TUPLETABLESLOT_ISNULL,
 214                                                                          "v_outernulls");
 215         v_resultvalues = l_load_struct_gep(b, v_resultslot,
 216                                                                            FIELDNO_TUPLETABLESLOT_VALUES,
 217                                                                            "v_resultvalues");
 218         v_resultnulls = l_load_struct_gep(b, v_resultslot,
 219                                                                           FIELDNO_TUPLETABLESLOT_ISNULL,
 220                                                                           "v_resultnulls");
 221
 222         /* aggvalues/aggnulls */
 223         v_aggvalues = l_load_struct_gep(b, v_econtext,
 224                                                                         FIELDNO_EXPRCONTEXT_AGGVALUES,
 225                                                                         "v.econtext.aggvalues");
 226         v_aggnulls = l_load_struct_gep(b, v_econtext,
 227                                                                    FIELDNO_EXPRCONTEXT_AGGNULLS,
 228                                                                    "v.econtext.aggnulls");
 229
 230         /* allocate blocks for each op upfront, so we can do jumps easily */
 231         opblocks = palloc(sizeof(LLVMBasicBlockRef) * state->steps_len);
 232         for (i = 0; i < state->steps_len; i++)
 233                 opblocks[i] = l_bb_append_v(eval_fn, "b.op.%d.start", i);
 234
 235         /* jump from entry to first block */
 236         LLVMBuildBr(b, opblocks[0]);
 237
 238         for (i = 0; i < state->steps_len; i++)
 239         {
 240                 ExprEvalStep *op;
 241                 ExprEvalOp      opcode;
 242                 LLVMValueRef v_resvaluep;
 243                 LLVMValueRef v_resnullp;
 244
 245                 LLVMPositionBuilderAtEnd(b, opblocks[i]);
 246
 247                 op = &state->steps[i];
 248                 opcode = ExecEvalStepOp(state, op);
 249
 250                 v_resvaluep = l_ptr_const(op->resvalue, l_ptr(TypeSizeT));
 251                 v_resnullp = l_ptr_const(op->resnull, l_ptr(TypeStorageBool));
 252
 253                 switch (opcode)
 254                 {
 255                         case EEOP_DONE:
 256                                 {
 257                                         LLVMValueRef v_tmpisnull,
 258                                                                 v_tmpvalue;
 259
 260                                         v_tmpvalue = LLVMBuildLoad(b, v_tmpvaluep, "");
 261                                         v_tmpisnull = LLVMBuildLoad(b, v_tmpisnullp, "");
 262                                         v_tmpisnull =
 263                                                 LLVMBuildTrunc(b, v_tmpisnull, TypeParamBool, "");
 264
 265                                         LLVMBuildStore(b, v_tmpisnull, v_isnullp);
 266
 267                                         LLVMBuildRet(b, v_tmpvalue);
 268                                         break;
 269                                 }
 270
 271                         case EEOP_INNER_FETCHSOME:
 272                         case EEOP_OUTER_FETCHSOME:
 273                         case EEOP_SCAN_FETCHSOME:
 274                                 {
 275                                         TupleDesc       desc = NULL;
 276                                         LLVMValueRef v_slot;
 277                                         LLVMBasicBlockRef b_fetch;
 278                                         LLVMValueRef v_nvalid;
 279                                         LLVMValueRef l_jit_deform = NULL;
 280                                         const TupleTableSlotOps *tts_ops = NULL;
 281
 282                                         b_fetch = l_bb_before_v(opblocks[i + 1],
 283                                                                                         "op.%d.fetch", i);
 284
 285                                         if (op->d.fetch.known_desc)
 286                                                 desc = op->d.fetch.known_desc;
 287
 288                                         if (op->d.fetch.fixed)
 289                                                 tts_ops = op->d.fetch.kind;
 290
 291                                         if (opcode == EEOP_INNER_FETCHSOME)
 292                                                 v_slot = v_innerslot;
 293                                         else if (opcode == EEOP_OUTER_FETCHSOME)
 294                                                 v_slot = v_outerslot;
 295                                         else
 296                                                 v_slot = v_scanslot;
 297
 298                                         /*
 299                                          * Check if all required attributes are available, or
 300                                          * whether deforming is required.
 301                                          *
 302                                          * TODO: skip nvalid check if slot is fixed and known to
 303                                          * be a virtual slot.
 304                                          */
 305                                         v_nvalid =
 306                                                 l_load_struct_gep(b, v_slot,
 307                                                                                   FIELDNO_TUPLETABLESLOT_NVALID,
 308                                                                                   "");
 309                                         LLVMBuildCondBr(b,
 310                                                                         LLVMBuildICmp(b, LLVMIntUGE, v_nvalid,
 311                                                                                                   l_int16_const(op->d.fetch.last_var),
 312                                                                                                   ""),
 313                                                                         opblocks[i + 1], b_fetch);
 314
 315                                         LLVMPositionBuilderAtEnd(b, b_fetch);
 316
 317                                         /*
 318                                          * If the tupledesc of the to-be-deformed tuple is known,
 319                                          * and JITing of deforming is enabled, build deform
 320                                          * function specific to tupledesc and the exact number of
 321                                          * to-be-extracted attributes.
 322                                          */
 323                                         if (tts_ops && desc && (context->base.flags & PGJIT_DEFORM))
 324                                         {
 325                                                 l_jit_deform =
 326                                                         slot_compile_deform(context, desc,
 327                                                                                                 tts_ops,
 328                                                                                                 op->d.fetch.last_var);
 329                                         }
 330
 331                                         if (l_jit_deform)
 332                                         {
 333                                                 LLVMValueRef params[1];
 334
 335                                                 params[0] = v_slot;
 336
 337                                                 LLVMBuildCall(b, l_jit_deform,
 338                                                                           params, lengthof(params), "");
 339                                         }
 340                                         else
 341                                         {
 342                                                 LLVMValueRef params[2];
 343
 344                                                 params[0] = v_slot;
 345                                                 params[1] = l_int32_const(op->d.fetch.last_var);
 346
 347                                                 LLVMBuildCall(b,
 348                                                                           llvm_get_decl(mod, FuncSlotGetsomeattrsInt),
 349                                                                           params, lengthof(params), "");
 350                                         }
 351
 352                                         LLVMBuildBr(b, opblocks[i + 1]);
 353                                         break;
 354                                 }
 355
 356                         case EEOP_INNER_VAR:
 357                         case EEOP_OUTER_VAR:
 358                         case EEOP_SCAN_VAR:
 359                                 {
 360                                         LLVMValueRef value,
 361                                                                 isnull;
 362                                         LLVMValueRef v_attnum;
 363                                         LLVMValueRef v_values;
 364                                         LLVMValueRef v_nulls;
 365
 366                                         if (opcode == EEOP_INNER_VAR)
 367                                         {
 368                                                 v_values = v_innervalues;
 369                                                 v_nulls = v_innernulls;
 370                                         }
 371                                         else if (opcode == EEOP_OUTER_VAR)
 372                                         {
 373                                                 v_values = v_outervalues;
 374                                                 v_nulls = v_outernulls;
 375                                         }
 376                                         else
 377                                         {
 378                                                 v_values = v_scanvalues;
 379                                                 v_nulls = v_scannulls;
 380                                         }
 381
 382                                         v_attnum = l_int32_const(op->d.var.attnum);
 383                                         value = l_load_gep1(b, v_values, v_attnum, "");
 384                                         isnull = l_load_gep1(b, v_nulls, v_attnum, "");
 385                                         LLVMBuildStore(b, value, v_resvaluep);
 386                                         LLVMBuildStore(b, isnull, v_resnullp);
 387
 388                                         LLVMBuildBr(b, opblocks[i + 1]);
 389                                         break;
 390                                 }
 391
 392                         case EEOP_INNER_SYSVAR:
 393                         case EEOP_OUTER_SYSVAR:
 394                         case EEOP_SCAN_SYSVAR:
 395                                 {
 396                                         LLVMValueRef v_slot;
 397                                         LLVMValueRef v_params[4];
 398
 399                                         if (opcode == EEOP_INNER_SYSVAR)
 400                                                 v_slot = v_innerslot;
 401                                         else if (opcode == EEOP_OUTER_SYSVAR)
 402                                                 v_slot = v_outerslot;
 403                                         else
 404                                                 v_slot = v_scanslot;
 405
 406                                         v_params[0] = v_state;
 407                                         v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
 408                                         v_params[2] = v_econtext;
 409                                         v_params[3] = v_slot;
 410
 411                                         LLVMBuildCall(b,
 412                                                                   llvm_get_decl(mod, FuncExecEvalSysVar),
 413                                                                   v_params, lengthof(v_params), "");
 414
 415                                         LLVMBuildBr(b, opblocks[i + 1]);
 416                                         break;
 417                                 }
 418
 419                         case EEOP_WHOLEROW:
 420                                 build_EvalXFunc(b, mod, "ExecEvalWholeRowVar",
 421                                                                 v_state, v_econtext, op);
 422                                 LLVMBuildBr(b, opblocks[i + 1]);
 423                                 break;
 424
 425                         case EEOP_ASSIGN_INNER_VAR:
 426                         case EEOP_ASSIGN_OUTER_VAR:
 427                         case EEOP_ASSIGN_SCAN_VAR:
 428                                 {
 429                                         LLVMValueRef v_value,
 430                                                                 v_isnull;
 431                                         LLVMValueRef v_rvaluep,
 432                                                                 v_risnullp;
 433                                         LLVMValueRef v_attnum,
 434                                                                 v_resultnum;
 435                                         LLVMValueRef v_values;
 436                                         LLVMValueRef v_nulls;
 437
 438                                         if (opcode == EEOP_ASSIGN_INNER_VAR)
 439                                         {
 440                                                 v_values = v_innervalues;
 441                                                 v_nulls = v_innernulls;
 442                                         }
 443                                         else if (opcode == EEOP_ASSIGN_OUTER_VAR)
 444                                         {
 445                                                 v_values = v_outervalues;
 446                                                 v_nulls = v_outernulls;
 447                                         }
 448                                         else
 449                                         {
 450                                                 v_values = v_scanvalues;
 451                                                 v_nulls = v_scannulls;
 452                                         }
 453
 454                                         /* load data */
 455                                         v_attnum = l_int32_const(op->d.assign_var.attnum);
 456                                         v_value = l_load_gep1(b, v_values, v_attnum, "");
 457                                         v_isnull = l_load_gep1(b, v_nulls, v_attnum, "");
 458
 459                                         /* compute addresses of targets */
 460                                         v_resultnum = l_int32_const(op->d.assign_var.resultnum);
 461                                         v_rvaluep = LLVMBuildGEP(b, v_resultvalues,
 462                                                                                          &v_resultnum, 1, "");
 463                                         v_risnullp = LLVMBuildGEP(b, v_resultnulls,
 464                                                                                           &v_resultnum, 1, "");
 465
 466                                         /* and store */
 467                                         LLVMBuildStore(b, v_value, v_rvaluep);
 468                                         LLVMBuildStore(b, v_isnull, v_risnullp);
 469
 470                                         LLVMBuildBr(b, opblocks[i + 1]);
 471                                         break;
 472                                 }
 473
 474                         case EEOP_ASSIGN_TMP:
 475                                 {
 476                                         LLVMValueRef v_value,
 477                                                                 v_isnull;
 478                                         LLVMValueRef v_rvaluep,
 479                                                                 v_risnullp;
 480                                         LLVMValueRef v_resultnum;
 481                                         size_t          resultnum = op->d.assign_tmp.resultnum;
 482
 483                                         /* load data */
 484                                         v_value = LLVMBuildLoad(b, v_tmpvaluep, "");
 485                                         v_isnull = LLVMBuildLoad(b, v_tmpisnullp, "");
 486
 487                                         /* compute addresses of targets */
 488                                         v_resultnum = l_int32_const(resultnum);
 489                                         v_rvaluep =
 490                                                 LLVMBuildGEP(b, v_resultvalues, &v_resultnum, 1, "");
 491                                         v_risnullp =
 492                                                 LLVMBuildGEP(b, v_resultnulls, &v_resultnum, 1, "");
 493
 494                                         /* and store */
 495                                         LLVMBuildStore(b, v_value, v_rvaluep);
 496                                         LLVMBuildStore(b, v_isnull, v_risnullp);
 497
 498                                         LLVMBuildBr(b, opblocks[i + 1]);
 499                                         break;
 500                                 }
 501
 502                         case EEOP_ASSIGN_TMP_MAKE_RO:
 503                                 {
 504                                         LLVMBasicBlockRef b_notnull;
 505                                         LLVMValueRef v_params[1];
 506                                         LLVMValueRef v_ret;
 507                                         LLVMValueRef v_value,
 508                                                                 v_isnull;
 509                                         LLVMValueRef v_rvaluep,
 510                                                                 v_risnullp;
 511                                         LLVMValueRef v_resultnum;
 512                                         size_t          resultnum = op->d.assign_tmp.resultnum;
 513
 514                                         b_notnull = l_bb_before_v(opblocks[i + 1],
 515                                                                                           "op.%d.assign_tmp.notnull", i);
 516
 517                                         /* load data */
 518                                         v_value = LLVMBuildLoad(b, v_tmpvaluep, "");
 519                                         v_isnull = LLVMBuildLoad(b, v_tmpisnullp, "");
 520
 521                                         /* compute addresses of targets */
 522                                         v_resultnum = l_int32_const(resultnum);
 523                                         v_rvaluep = LLVMBuildGEP(b, v_resultvalues,
 524                                                                                          &v_resultnum, 1, "");
 525                                         v_risnullp = LLVMBuildGEP(b, v_resultnulls,
 526                                                                                           &v_resultnum, 1, "");
 527
 528                                         /* store nullness */
 529                                         LLVMBuildStore(b, v_isnull, v_risnullp);
 530
 531                                         /* check if value is NULL */
 532                                         LLVMBuildCondBr(b,
 533                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_isnull,
 534                                                                                                   l_sbool_const(0), ""),
 535                                                                         b_notnull, opblocks[i + 1]);
 536
 537                                         /* if value is not null, convert to RO datum */
 538                                         LLVMPositionBuilderAtEnd(b, b_notnull);
 539                                         v_params[0] = v_value;
 540                                         v_ret =
 541                                                 LLVMBuildCall(b,
 542                                                                           llvm_get_decl(mod, FuncMakeExpandedObjectReadOnlyInternal),
 543                                                                           v_params, lengthof(v_params), "");
 544
 545                                         /* store value */
 546                                         LLVMBuildStore(b, v_ret, v_rvaluep);
 547
 548                                         LLVMBuildBr(b, opblocks[i + 1]);
 549                                         break;
 550                                 }
 551
 552                         case EEOP_CONST:
 553                                 {
 554                                         LLVMValueRef v_constvalue,
 555                                                                 v_constnull;
 556
 557                                         v_constvalue = l_sizet_const(op->d.constval.value);
 558                                         v_constnull = l_sbool_const(op->d.constval.isnull);
 559
 560                                         LLVMBuildStore(b, v_constvalue, v_resvaluep);
 561                                         LLVMBuildStore(b, v_constnull, v_resnullp);
 562
 563                                         LLVMBuildBr(b, opblocks[i + 1]);
 564                                         break;
 565                                 }
 566
 567                         case EEOP_FUNCEXPR_STRICT:
 568                                 {
 569                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
 570                                         LLVMBasicBlockRef b_nonull;
 571                                         int                     argno;
 572                                         LLVMValueRef v_fcinfo;
 573                                         LLVMBasicBlockRef *b_checkargnulls;
 574
 575                                         /*
 576                                          * Block for the actual function call, if args are
 577                                          * non-NULL.
 578                                          */
 579                                         b_nonull = l_bb_before_v(opblocks[i + 1],
 580                                                                                          "b.%d.no-null-args", i);
 581
 582                                         /* should make sure they're optimized beforehand */
 583                                         if (op->d.func.nargs == 0)
 584                                                 elog(ERROR, "argumentless strict functions are pointless");
 585
 586                                         v_fcinfo =
 587                                                 l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
 588
 589                                         /*
 590                                          * set resnull to true, if the function is actually
 591                                          * called, it'll be reset
 592                                          */
 593                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
 594
 595                                         /* create blocks for checking args, one for each */
 596                                         b_checkargnulls =
 597                                                 palloc(sizeof(LLVMBasicBlockRef *) * op->d.func.nargs);
 598                                         for (argno = 0; argno < op->d.func.nargs; argno++)
 599                                                 b_checkargnulls[argno] =
 600                                                         l_bb_before_v(b_nonull, "b.%d.isnull.%d", i, argno);
 601
 602                                         /* jump to check of first argument */
 603                                         LLVMBuildBr(b, b_checkargnulls[0]);
 604
 605                                         /* check each arg for NULLness */
 606                                         for (argno = 0; argno < op->d.func.nargs; argno++)
 607                                         {
 608                                                 LLVMValueRef v_argisnull;
 609                                                 LLVMBasicBlockRef b_argnotnull;
 610
 611                                                 LLVMPositionBuilderAtEnd(b, b_checkargnulls[argno]);
 612
 613                                                 /* compute block to jump to if argument is not null */
 614                                                 if (argno + 1 == op->d.func.nargs)
 615                                                         b_argnotnull = b_nonull;
 616                                                 else
 617                                                         b_argnotnull = b_checkargnulls[argno + 1];
 618
 619                                                 /* and finally load & check NULLness of arg */
 620                                                 v_argisnull = l_funcnull(b, v_fcinfo, argno);
 621                                                 LLVMBuildCondBr(b,
 622                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
 623                                                                                                           v_argisnull,
 624                                                                                                           l_sbool_const(1),
 625                                                                                                           ""),
 626                                                                                 opblocks[i + 1],
 627                                                                                 b_argnotnull);
 628                                         }
 629
 630                                         LLVMPositionBuilderAtEnd(b, b_nonull);
 631                                 }
 632                                 /* FALLTHROUGH */
 633
 634                         case EEOP_FUNCEXPR:
 635                                 {
 636                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
 637                                         LLVMValueRef v_fcinfo_isnull;
 638                                         LLVMValueRef v_retval;
 639
 640                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
 641                                                                                    &v_fcinfo_isnull);
 642                                         LLVMBuildStore(b, v_retval, v_resvaluep);
 643                                         LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp);
 644
 645                                         LLVMBuildBr(b, opblocks[i + 1]);
 646                                         break;
 647                                 }
 648
 649                         case EEOP_FUNCEXPR_FUSAGE:
 650                                 build_EvalXFunc(b, mod, "ExecEvalFuncExprFusage",
 651                                                                 v_state, v_econtext, op);
 652                                 LLVMBuildBr(b, opblocks[i + 1]);
 653                                 break;
 654
 655
 656                         case EEOP_FUNCEXPR_STRICT_FUSAGE:
 657                                 build_EvalXFunc(b, mod, "ExecEvalFuncExprStrictFusage",
 658                                                                 v_state, v_econtext, op);
 659                                 LLVMBuildBr(b, opblocks[i + 1]);
 660                                 break;
 661
 662                         case EEOP_BOOL_AND_STEP_FIRST:
 663                                 {
 664                                         LLVMValueRef v_boolanynullp;
 665
 666                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 667                                                                                                  l_ptr(TypeStorageBool));
 668                                         LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp);
 669
 670                                 }
 671                                 /* FALLTHROUGH */
 672
 673                                 /*
 674                                  * Treat them the same for now, optimizer can remove
 675                                  * redundancy. Could be worthwhile to optimize during emission
 676                                  * though.
 677                                  */
 678                         case EEOP_BOOL_AND_STEP_LAST:
 679                         case EEOP_BOOL_AND_STEP:
 680                                 {
 681                                         LLVMValueRef v_boolvalue;
 682                                         LLVMValueRef v_boolnull;
 683                                         LLVMValueRef v_boolanynullp,
 684                                                                 v_boolanynull;
 685                                         LLVMBasicBlockRef b_boolisnull;
 686                                         LLVMBasicBlockRef b_boolcheckfalse;
 687                                         LLVMBasicBlockRef b_boolisfalse;
 688                                         LLVMBasicBlockRef b_boolcont;
 689                                         LLVMBasicBlockRef b_boolisanynull;
 690
 691                                         b_boolisnull = l_bb_before_v(opblocks[i + 1],
 692                                                                                                  "b.%d.boolisnull", i);
 693                                         b_boolcheckfalse = l_bb_before_v(opblocks[i + 1],
 694                                                                                                          "b.%d.boolcheckfalse", i);
 695                                         b_boolisfalse = l_bb_before_v(opblocks[i + 1],
 696                                                                                                   "b.%d.boolisfalse", i);
 697                                         b_boolisanynull = l_bb_before_v(opblocks[i + 1],
 698                                                                                                         "b.%d.boolisanynull", i);
 699                                         b_boolcont = l_bb_before_v(opblocks[i + 1],
 700                                                                                            "b.%d.boolcont", i);
 701
 702                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 703                                                                                                  l_ptr(TypeStorageBool));
 704
 705                                         v_boolnull = LLVMBuildLoad(b, v_resnullp, "");
 706                                         v_boolvalue = LLVMBuildLoad(b, v_resvaluep, "");
 707
 708                                         /* set resnull to boolnull */
 709                                         LLVMBuildStore(b, v_boolnull, v_resnullp);
 710                                         /* set revalue to boolvalue */
 711                                         LLVMBuildStore(b, v_boolvalue, v_resvaluep);
 712
 713                                         /* check if current input is NULL */
 714                                         LLVMBuildCondBr(b,
 715                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolnull,
 716                                                                                                   l_sbool_const(1), ""),
 717                                                                         b_boolisnull,
 718                                                                         b_boolcheckfalse);
 719
 720                                         /* build block that sets anynull */
 721                                         LLVMPositionBuilderAtEnd(b, b_boolisnull);
 722                                         /* set boolanynull to true */
 723                                         LLVMBuildStore(b, l_sbool_const(1), v_boolanynullp);
 724                                         /* and jump to next block */
 725                                         LLVMBuildBr(b, b_boolcont);
 726
 727                                         /* build block checking for false */
 728                                         LLVMPositionBuilderAtEnd(b, b_boolcheckfalse);
 729                                         LLVMBuildCondBr(b,
 730                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolvalue,
 731                                                                                                   l_sizet_const(0), ""),
 732                                                                         b_boolisfalse,
 733                                                                         b_boolcont);
 734
 735                                         /*
 736                                          * Build block handling FALSE. Value is false, so short
 737                                          * circuit.
 738                                          */
 739                                         LLVMPositionBuilderAtEnd(b, b_boolisfalse);
 740                                         /* result is already set to FALSE, need not change it */
 741                                         /* and jump to the end of the AND expression */
 742                                         LLVMBuildBr(b, opblocks[op->d.boolexpr.jumpdone]);
 743
 744                                         /* Build block that continues if bool is TRUE. */
 745                                         LLVMPositionBuilderAtEnd(b, b_boolcont);
 746
 747                                         v_boolanynull = LLVMBuildLoad(b, v_boolanynullp, "");
 748
 749                                         /* set value to NULL if any previous values were NULL */
 750                                         LLVMBuildCondBr(b,
 751                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolanynull,
 752                                                                                                   l_sbool_const(0), ""),
 753                                                                         opblocks[i + 1], b_boolisanynull);
 754
 755                                         LLVMPositionBuilderAtEnd(b, b_boolisanynull);
 756                                         /* set resnull to true */
 757                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
 758                                         /* reset resvalue */
 759                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
 760
 761                                         LLVMBuildBr(b, opblocks[i + 1]);
 762                                         break;
 763                                 }
 764                         case EEOP_BOOL_OR_STEP_FIRST:
 765                                 {
 766                                         LLVMValueRef v_boolanynullp;
 767
 768                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 769                                                                                                  l_ptr(TypeStorageBool));
 770                                         LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp);
 771                                 }
 772                                 /* FALLTHROUGH */
 773
 774                                 /*
 775                                  * Treat them the same for now, optimizer can remove
 776                                  * redundancy. Could be worthwhile to optimize during emission
 777                                  * though.
 778                                  */
 779                         case EEOP_BOOL_OR_STEP_LAST:
 780                         case EEOP_BOOL_OR_STEP:
 781                                 {
 782                                         LLVMValueRef v_boolvalue;
 783                                         LLVMValueRef v_boolnull;
 784                                         LLVMValueRef v_boolanynullp,
 785                                                                 v_boolanynull;
 786
 787                                         LLVMBasicBlockRef b_boolisnull;
 788                                         LLVMBasicBlockRef b_boolchecktrue;
 789                                         LLVMBasicBlockRef b_boolistrue;
 790                                         LLVMBasicBlockRef b_boolcont;
 791                                         LLVMBasicBlockRef b_boolisanynull;
 792
 793                                         b_boolisnull = l_bb_before_v(opblocks[i + 1],
 794                                                                                                  "b.%d.boolisnull", i);
 795                                         b_boolchecktrue = l_bb_before_v(opblocks[i + 1],
 796                                                                                                         "b.%d.boolchecktrue", i);
 797                                         b_boolistrue = l_bb_before_v(opblocks[i + 1],
 798                                                                                                  "b.%d.boolistrue", i);
 799                                         b_boolisanynull = l_bb_before_v(opblocks[i + 1],
 800                                                                                                         "b.%d.boolisanynull", i);
 801                                         b_boolcont = l_bb_before_v(opblocks[i + 1],
 802                                                                                            "b.%d.boolcont", i);
 803
 804                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 805                                                                                                  l_ptr(TypeStorageBool));
 806
 807                                         v_boolnull = LLVMBuildLoad(b, v_resnullp, "");
 808                                         v_boolvalue = LLVMBuildLoad(b, v_resvaluep, "");
 809
 810                                         /* set resnull to boolnull */
 811                                         LLVMBuildStore(b, v_boolnull, v_resnullp);
 812                                         /* set revalue to boolvalue */
 813                                         LLVMBuildStore(b, v_boolvalue, v_resvaluep);
 814
 815                                         LLVMBuildCondBr(b,
 816                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolnull,
 817                                                                                                   l_sbool_const(1), ""),
 818                                                                         b_boolisnull,
 819                                                                         b_boolchecktrue);
 820
 821                                         /* build block that sets anynull */
 822                                         LLVMPositionBuilderAtEnd(b, b_boolisnull);
 823                                         /* set boolanynull to true */
 824                                         LLVMBuildStore(b, l_sbool_const(1), v_boolanynullp);
 825                                         /* and jump to next block */
 826                                         LLVMBuildBr(b, b_boolcont);
 827
 828                                         /* build block checking for true */
 829                                         LLVMPositionBuilderAtEnd(b, b_boolchecktrue);
 830                                         LLVMBuildCondBr(b,
 831                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolvalue,
 832                                                                                                   l_sizet_const(1), ""),
 833                                                                         b_boolistrue,
 834                                                                         b_boolcont);
 835
 836                                         /*
 837                                          * Build block handling True. Value is true, so short
 838                                          * circuit.
 839                                          */
 840                                         LLVMPositionBuilderAtEnd(b, b_boolistrue);
 841                                         /* result is already set to TRUE, need not change it */
 842                                         /* and jump to the end of the OR expression */
 843                                         LLVMBuildBr(b, opblocks[op->d.boolexpr.jumpdone]);
 844
 845                                         /* build block that continues if bool is FALSE */
 846                                         LLVMPositionBuilderAtEnd(b, b_boolcont);
 847
 848                                         v_boolanynull = LLVMBuildLoad(b, v_boolanynullp, "");
 849
 850                                         /* set value to NULL if any previous values were NULL */
 851                                         LLVMBuildCondBr(b,
 852                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolanynull,
 853                                                                                                   l_sbool_const(0), ""),
 854                                                                         opblocks[i + 1], b_boolisanynull);
 855
 856                                         LLVMPositionBuilderAtEnd(b, b_boolisanynull);
 857                                         /* set resnull to true */
 858                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
 859                                         /* reset resvalue */
 860                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
 861
 862                                         LLVMBuildBr(b, opblocks[i + 1]);
 863                                         break;
 864                                 }
 865
 866                         case EEOP_BOOL_NOT_STEP:
 867                                 {
 868                                         LLVMValueRef v_boolvalue;
 869                                         LLVMValueRef v_boolnull;
 870                                         LLVMValueRef v_negbool;
 871
 872                                         v_boolnull = LLVMBuildLoad(b, v_resnullp, "");
 873                                         v_boolvalue = LLVMBuildLoad(b, v_resvaluep, "");
 874
 875                                         v_negbool = LLVMBuildZExt(b,
 876                                                                                           LLVMBuildICmp(b, LLVMIntEQ,
 877                                                                                                                         v_boolvalue,
 878                                                                                                                         l_sizet_const(0),
 879                                                                                                                         ""),
 880                                                                                           TypeSizeT, "");
 881                                         /* set resnull to boolnull */
 882                                         LLVMBuildStore(b, v_boolnull, v_resnullp);
 883                                         /* set revalue to !boolvalue */
 884                                         LLVMBuildStore(b, v_negbool, v_resvaluep);
 885
 886                                         LLVMBuildBr(b, opblocks[i + 1]);
 887                                         break;
 888                                 }
 889
 890                         case EEOP_QUAL:
 891                                 {
 892                                         LLVMValueRef v_resnull;
 893                                         LLVMValueRef v_resvalue;
 894                                         LLVMValueRef v_nullorfalse;
 895                                         LLVMBasicBlockRef b_qualfail;
 896
 897                                         b_qualfail = l_bb_before_v(opblocks[i + 1],
 898                                                                                            "op.%d.qualfail", i);
 899
 900                                         v_resvalue = LLVMBuildLoad(b, v_resvaluep, "");
 901                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 902
 903                                         v_nullorfalse =
 904                                                 LLVMBuildOr(b,
 905                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
 906                                                                                                   l_sbool_const(1), ""),
 907                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resvalue,
 908                                                                                                   l_sizet_const(0), ""),
 909                                                                         "");
 910
 911                                         LLVMBuildCondBr(b,
 912                                                                         v_nullorfalse,
 913                                                                         b_qualfail,
 914                                                                         opblocks[i + 1]);
 915
 916                                         /* build block handling NULL or false */
 917                                         LLVMPositionBuilderAtEnd(b, b_qualfail);
 918                                         /* set resnull to false */
 919                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
 920                                         /* set resvalue to false */
 921                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
 922                                         /* and jump out */
 923                                         LLVMBuildBr(b, opblocks[op->d.qualexpr.jumpdone]);
 924                                         break;
 925                                 }
 926
 927                         case EEOP_JUMP:
 928                                 {
 929                                         LLVMBuildBr(b, opblocks[op->d.jump.jumpdone]);
 930                                         break;
 931                                 }
 932
 933                         case EEOP_JUMP_IF_NULL:
 934                                 {
 935                                         LLVMValueRef v_resnull;
 936
 937                                         /* Transfer control if current result is null */
 938
 939                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 940
 941                                         LLVMBuildCondBr(b,
 942                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
 943                                                                                                   l_sbool_const(1), ""),
 944                                                                         opblocks[op->d.jump.jumpdone],
 945                                                                         opblocks[i + 1]);
 946                                         break;
 947                                 }
 948
 949                         case EEOP_JUMP_IF_NOT_NULL:
 950                                 {
 951                                         LLVMValueRef v_resnull;
 952
 953                                         /* Transfer control if current result is non-null */
 954
 955                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 956
 957                                         LLVMBuildCondBr(b,
 958                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
 959                                                                                                   l_sbool_const(0), ""),
 960                                                                         opblocks[op->d.jump.jumpdone],
 961                                                                         opblocks[i + 1]);
 962                                         break;
 963                                 }
 964
 965
 966                         case EEOP_JUMP_IF_NOT_TRUE:
 967                                 {
 968                                         LLVMValueRef v_resnull;
 969                                         LLVMValueRef v_resvalue;
 970                                         LLVMValueRef v_nullorfalse;
 971
 972                                         /* Transfer control if current result is null or false */
 973
 974                                         v_resvalue = LLVMBuildLoad(b, v_resvaluep, "");
 975                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 976
 977                                         v_nullorfalse =
 978                                                 LLVMBuildOr(b,
 979                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
 980                                                                                                   l_sbool_const(1), ""),
 981                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resvalue,
 982                                                                                                   l_sizet_const(0), ""),
 983                                                                         "");
 984
 985                                         LLVMBuildCondBr(b,
 986                                                                         v_nullorfalse,
 987                                                                         opblocks[op->d.jump.jumpdone],
 988                                                                         opblocks[i + 1]);
 989                                         break;
 990                                 }
 991
 992                         case EEOP_NULLTEST_ISNULL:
 993                                 {
 994                                         LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 995                                         LLVMValueRef v_resvalue;
 996
 997                                         v_resvalue =
 998                                                 LLVMBuildSelect(b,
 999                                                                                 LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1000                                                                                                           l_sbool_const(1), ""),
1001                                                                                 l_sizet_const(1),
1002                                                                                 l_sizet_const(0),
1003                                                                                 "");
1004                                         LLVMBuildStore(b, v_resvalue, v_resvaluep);
1005                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1006
1007                                         LLVMBuildBr(b, opblocks[i + 1]);
1008                                         break;
1009                                 }
1010
1011                         case EEOP_NULLTEST_ISNOTNULL:
1012                                 {
1013                                         LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1014                                         LLVMValueRef v_resvalue;
1015
1016                                         v_resvalue =
1017                                                 LLVMBuildSelect(b,
1018                                                                                 LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1019                                                                                                           l_sbool_const(1), ""),
1020                                                                                 l_sizet_const(0),
1021                                                                                 l_sizet_const(1),
1022                                                                                 "");
1023                                         LLVMBuildStore(b, v_resvalue, v_resvaluep);
1024                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1025
1026                                         LLVMBuildBr(b, opblocks[i + 1]);
1027                                         break;
1028                                 }
1029
1030                         case EEOP_NULLTEST_ROWISNULL:
1031                                 build_EvalXFunc(b, mod, "ExecEvalRowNull",
1032                                                                 v_state, v_econtext, op);
1033                                 LLVMBuildBr(b, opblocks[i + 1]);
1034                                 break;
1035
1036                         case EEOP_NULLTEST_ROWISNOTNULL:
1037                                 build_EvalXFunc(b, mod, "ExecEvalRowNotNull",
1038                                                                 v_state, v_econtext, op);
1039                                 LLVMBuildBr(b, opblocks[i + 1]);
1040                                 break;
1041
1042                         case EEOP_BOOLTEST_IS_TRUE:
1043                         case EEOP_BOOLTEST_IS_NOT_FALSE:
1044                         case EEOP_BOOLTEST_IS_FALSE:
1045                         case EEOP_BOOLTEST_IS_NOT_TRUE:
1046                                 {
1047                                         LLVMBasicBlockRef b_isnull,
1048                                                                 b_notnull;
1049                                         LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1050
1051                                         b_isnull = l_bb_before_v(opblocks[i + 1],
1052                                                                                          "op.%d.isnull", i);
1053                                         b_notnull = l_bb_before_v(opblocks[i + 1],
1054                                                                                           "op.%d.isnotnull", i);
1055
1056                                         /* check if value is NULL */
1057                                         LLVMBuildCondBr(b,
1058                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1059                                                                                                   l_sbool_const(1), ""),
1060                                                                         b_isnull, b_notnull);
1061
1062                                         /* if value is NULL, return false */
1063                                         LLVMPositionBuilderAtEnd(b, b_isnull);
1064
1065                                         /* result is not null */
1066                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1067
1068                                         if (opcode == EEOP_BOOLTEST_IS_TRUE ||
1069                                                 opcode == EEOP_BOOLTEST_IS_FALSE)
1070                                         {
1071                                                 LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1072                                         }
1073                                         else
1074                                         {
1075                                                 LLVMBuildStore(b, l_sizet_const(1), v_resvaluep);
1076                                         }
1077
1078                                         LLVMBuildBr(b, opblocks[i + 1]);
1079
1080                                         LLVMPositionBuilderAtEnd(b, b_notnull);
1081
1082                                         if (opcode == EEOP_BOOLTEST_IS_TRUE ||
1083                                                 opcode == EEOP_BOOLTEST_IS_NOT_FALSE)
1084                                         {
1085                                                 /*
1086                                                  * if value is not null NULL, return value (already
1087                                                  * set)
1088                                                  */
1089                                         }
1090                                         else
1091                                         {
1092                                                 LLVMValueRef v_value =
1093                                                 LLVMBuildLoad(b, v_resvaluep, "");
1094
1095                                                 v_value = LLVMBuildZExt(b,
1096                                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
1097                                                                                                                           v_value,
1098                                                                                                                           l_sizet_const(0),
1099                                                                                                                           ""),
1100                                                                                                 TypeSizeT, "");
1101                                                 LLVMBuildStore(b, v_value, v_resvaluep);
1102                                         }
1103                                         LLVMBuildBr(b, opblocks[i + 1]);
1104                                         break;
1105                                 }
1106
1107                         case EEOP_PARAM_EXEC:
1108                                 build_EvalXFunc(b, mod, "ExecEvalParamExec",
1109                                                                 v_state, v_econtext, op);
1110                                 LLVMBuildBr(b, opblocks[i + 1]);
1111                                 break;
1112
1113                         case EEOP_PARAM_EXTERN:
1114                                 build_EvalXFunc(b, mod, "ExecEvalParamExtern",
1115                                                                 v_state, v_econtext, op);
1116                                 LLVMBuildBr(b, opblocks[i + 1]);
1117                                 break;
1118
1119                         case EEOP_PARAM_CALLBACK:
1120                                 {
1121                                         LLVMTypeRef param_types[3];
1122                                         LLVMValueRef v_params[3];
1123                                         LLVMTypeRef v_functype;
1124                                         LLVMValueRef v_func;
1125
1126                                         param_types[0] = l_ptr(StructExprState);
1127                                         param_types[1] = l_ptr(TypeSizeT);
1128                                         param_types[2] = l_ptr(StructExprContext);
1129
1130                                         v_functype = LLVMFunctionType(LLVMVoidType(),
1131                                                                                                   param_types,
1132                                                                                                   lengthof(param_types),
1133                                                                                                   false);
1134                                         v_func = l_ptr_const(op->d.cparam.paramfunc,
1135                                                                                  l_ptr(v_functype));
1136
1137                                         v_params[0] = v_state;
1138                                         v_params[1] = l_ptr_const(op, l_ptr(TypeSizeT));
1139                                         v_params[2] = v_econtext;
1140                                         LLVMBuildCall(b,
1141                                                                   v_func,
1142                                                                   v_params, lengthof(v_params), "");
1143
1144                                         LLVMBuildBr(b, opblocks[i + 1]);
1145                                         break;
1146                                 }
1147
1148                         case EEOP_ARRAYREF_OLD:
1149                                 build_EvalXFunc(b, mod, "ExecEvalArrayRefOld",
1150                                                                 v_state, v_econtext, op);
1151                                 LLVMBuildBr(b, opblocks[i + 1]);
1152                                 break;
1153
1154                         case EEOP_ARRAYREF_ASSIGN:
1155                                 build_EvalXFunc(b, mod, "ExecEvalArrayRefAssign",
1156                                                                 v_state, v_econtext, op);
1157                                 LLVMBuildBr(b, opblocks[i + 1]);
1158                                 break;
1159
1160                         case EEOP_ARRAYREF_FETCH:
1161                                 build_EvalXFunc(b, mod, "ExecEvalArrayRefFetch",
1162                                                                 v_state, v_econtext, op);
1163                                 LLVMBuildBr(b, opblocks[i + 1]);
1164                                 break;
1165
1166                         case EEOP_CASE_TESTVAL:
1167                                 {
1168                                         LLVMBasicBlockRef b_avail,
1169                                                                 b_notavail;
1170                                         LLVMValueRef v_casevaluep,
1171                                                                 v_casevalue;
1172                                         LLVMValueRef v_casenullp,
1173                                                                 v_casenull;
1174                                         LLVMValueRef v_casevaluenull;
1175
1176                                         b_avail = l_bb_before_v(opblocks[i + 1],
1177                                                                                         "op.%d.avail", i);
1178                                         b_notavail = l_bb_before_v(opblocks[i + 1],
1179                                                                                            "op.%d.notavail", i);
1180
1181                                         v_casevaluep = l_ptr_const(op->d.casetest.value,
1182                                                                                            l_ptr(TypeSizeT));
1183                                         v_casenullp = l_ptr_const(op->d.casetest.isnull,
1184                                                                                           l_ptr(TypeStorageBool));
1185
1186                                         v_casevaluenull =
1187                                                 LLVMBuildICmp(b, LLVMIntEQ,
1188                                                                           LLVMBuildPtrToInt(b, v_casevaluep,
1189                                                                                                                 TypeSizeT, ""),
1190                                                                           l_sizet_const(0), "");
1191                                         LLVMBuildCondBr(b, v_casevaluenull, b_notavail, b_avail);
1192
1193                                         /* if casetest != NULL */
1194                                         LLVMPositionBuilderAtEnd(b, b_avail);
1195                                         v_casevalue = LLVMBuildLoad(b, v_casevaluep, "");
1196                                         v_casenull = LLVMBuildLoad(b, v_casenullp, "");
1197                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1198                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1199                                         LLVMBuildBr(b, opblocks[i + 1]);
1200
1201                                         /* if casetest == NULL */
1202                                         LLVMPositionBuilderAtEnd(b, b_notavail);
1203                                         v_casevalue =
1204                                                 l_load_struct_gep(b, v_econtext,
1205                                                                                   FIELDNO_EXPRCONTEXT_CASEDATUM, "");
1206                                         v_casenull =
1207                                                 l_load_struct_gep(b, v_econtext,
1208                                                                                   FIELDNO_EXPRCONTEXT_CASENULL, "");
1209                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1210                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1211
1212                                         LLVMBuildBr(b, opblocks[i + 1]);
1213                                         break;
1214                                 }
1215
1216                         case EEOP_MAKE_READONLY:
1217                                 {
1218                                         LLVMBasicBlockRef b_notnull;
1219                                         LLVMValueRef v_params[1];
1220                                         LLVMValueRef v_ret;
1221                                         LLVMValueRef v_nullp;
1222                                         LLVMValueRef v_valuep;
1223                                         LLVMValueRef v_null;
1224                                         LLVMValueRef v_value;
1225
1226                                         b_notnull = l_bb_before_v(opblocks[i + 1],
1227                                                                                           "op.%d.readonly.notnull", i);
1228
1229                                         v_nullp = l_ptr_const(op->d.make_readonly.isnull,
1230                                                                                   l_ptr(TypeStorageBool));
1231
1232                                         v_null = LLVMBuildLoad(b, v_nullp, "");
1233
1234                                         /* store null isnull value in result */
1235                                         LLVMBuildStore(b, v_null, v_resnullp);
1236
1237                                         /* check if value is NULL */
1238                                         LLVMBuildCondBr(b,
1239                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_null,
1240                                                                                                   l_sbool_const(1), ""),
1241                                                                         opblocks[i + 1], b_notnull);
1242
1243                                         /* if value is not null, convert to RO datum */
1244                                         LLVMPositionBuilderAtEnd(b, b_notnull);
1245
1246                                         v_valuep = l_ptr_const(op->d.make_readonly.value,
1247                                                                                    l_ptr(TypeSizeT));
1248
1249                                         v_value = LLVMBuildLoad(b, v_valuep, "");
1250
1251                                         v_params[0] = v_value;
1252                                         v_ret =
1253                                                 LLVMBuildCall(b,
1254                                                                           llvm_get_decl(mod, FuncMakeExpandedObjectReadOnlyInternal),
1255                                                                           v_params, lengthof(v_params), "");
1256                                         LLVMBuildStore(b, v_ret, v_resvaluep);
1257
1258                                         LLVMBuildBr(b, opblocks[i + 1]);
1259                                         break;
1260                                 }
1261
1262                         case EEOP_IOCOERCE:
1263                                 {
1264                                         FunctionCallInfo fcinfo_out,
1265                                                                 fcinfo_in;
1266                                         LLVMValueRef v_fcinfo_out,
1267                                                                 v_fcinfo_in;
1268                                         LLVMValueRef v_fn_addr_out,
1269                                                                 v_fn_addr_in;
1270                                         LLVMValueRef v_fcinfo_in_isnullp;
1271                                         LLVMValueRef v_retval;
1272                                         LLVMValueRef v_resvalue;
1273                                         LLVMValueRef v_resnull;
1274
1275                                         LLVMValueRef v_output_skip;
1276                                         LLVMValueRef v_output;
1277
1278                                         LLVMBasicBlockRef b_skipoutput;
1279                                         LLVMBasicBlockRef b_calloutput;
1280                                         LLVMBasicBlockRef b_input;
1281                                         LLVMBasicBlockRef b_inputcall;
1282
1283                                         fcinfo_out = op->d.iocoerce.fcinfo_data_out;
1284                                         fcinfo_in = op->d.iocoerce.fcinfo_data_in;
1285
1286                                         b_skipoutput = l_bb_before_v(opblocks[i + 1],
1287                                                                                                  "op.%d.skipoutputnull", i);
1288                                         b_calloutput = l_bb_before_v(opblocks[i + 1],
1289                                                                                                  "op.%d.calloutput", i);
1290                                         b_input = l_bb_before_v(opblocks[i + 1],
1291                                                                                         "op.%d.input", i);
1292                                         b_inputcall = l_bb_before_v(opblocks[i + 1],
1293                                                                                                 "op.%d.inputcall", i);
1294
1295                                         v_fcinfo_out = l_ptr_const(fcinfo_out, l_ptr(StructFunctionCallInfoData));
1296                                         v_fcinfo_in = l_ptr_const(fcinfo_in, l_ptr(StructFunctionCallInfoData));
1297                                         v_fn_addr_out = l_ptr_const(fcinfo_out->flinfo->fn_addr, TypePGFunction);
1298                                         v_fn_addr_in = l_ptr_const(fcinfo_in->flinfo->fn_addr, TypePGFunction);
1299
1300                                         v_fcinfo_in_isnullp =
1301                                                 LLVMBuildStructGEP(b, v_fcinfo_in,
1302                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ISNULL,
1303                                                                                    "v_fcinfo_in_isnull");
1304
1305                                         /* output functions are not called on nulls */
1306                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1307                                         LLVMBuildCondBr(b,
1308                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1309                                                                                                   l_sbool_const(1), ""),
1310                                                                         b_skipoutput,
1311                                                                         b_calloutput);
1312
1313                                         LLVMPositionBuilderAtEnd(b, b_skipoutput);
1314                                         v_output_skip = l_sizet_const(0);
1315                                         LLVMBuildBr(b, b_input);
1316
1317                                         LLVMPositionBuilderAtEnd(b, b_calloutput);
1318                                         v_resvalue = LLVMBuildLoad(b, v_resvaluep, "");
1319
1320                                         /* set arg[0] */
1321                                         LLVMBuildStore(b,
1322                                                                    v_resvalue,
1323                                                                    l_funcvaluep(b, v_fcinfo_out, 0));
1324                                         LLVMBuildStore(b,
1325                                                                    l_sbool_const(0),
1326                                                                    l_funcnullp(b, v_fcinfo_out, 0));
1327                                         /* and call output function (can never return NULL) */
1328                                         v_output = LLVMBuildCall(b, v_fn_addr_out, &v_fcinfo_out,
1329                                                                                          1, "funccall_coerce_out");
1330                                         LLVMBuildBr(b, b_input);
1331
1332                                         /* build block handling input function call */
1333                                         LLVMPositionBuilderAtEnd(b, b_input);
1334
1335                                         /* phi between resnull and output function call branches */
1336                                         {
1337                                                 LLVMValueRef incoming_values[2];
1338                                                 LLVMBasicBlockRef incoming_blocks[2];
1339
1340                                                 incoming_values[0] = v_output_skip;
1341                                                 incoming_blocks[0] = b_skipoutput;
1342
1343                                                 incoming_values[1] = v_output;
1344                                                 incoming_blocks[1] = b_calloutput;
1345
1346                                                 v_output = LLVMBuildPhi(b, TypeSizeT, "output");
1347                                                 LLVMAddIncoming(v_output,
1348                                                                                 incoming_values, incoming_blocks,
1349                                                                                 lengthof(incoming_blocks));
1350                                         }
1351
1352                                         /*
1353                                          * If input function is strict, skip if input string is
1354                                          * NULL.
1355                                          */
1356                                         if (op->d.iocoerce.finfo_in->fn_strict)
1357                                         {
1358                                                 LLVMBuildCondBr(b,
1359                                                                                 LLVMBuildICmp(b, LLVMIntEQ, v_output,
1360                                                                                                           l_sizet_const(0), ""),
1361                                                                                 opblocks[i + 1],
1362                                                                                 b_inputcall);
1363                                         }
1364                                         else
1365                                         {
1366                                                 LLVMBuildBr(b, b_inputcall);
1367                                         }
1368
1369                                         LLVMPositionBuilderAtEnd(b, b_inputcall);
1370                                         /* set arguments */
1371                                         /* arg0: output */
1372                                         LLVMBuildStore(b, v_output,
1373                                                                    l_funcvaluep(b, v_fcinfo_in, 0));
1374                                         LLVMBuildStore(b, v_resnull,
1375                                                                    l_funcnullp(b, v_fcinfo_in, 0));
1376
1377                                         /* arg1: ioparam: preset in execExpr.c */
1378                                         /* arg2: typmod: preset in execExpr.c  */
1379
1380                                         /* reset fcinfo_in->isnull */
1381                                         LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_in_isnullp);
1382                                         /* and call function */
1383                                         v_retval = LLVMBuildCall(b, v_fn_addr_in, &v_fcinfo_in, 1,
1384                                                                                          "funccall_iocoerce_in");
1385
1386                                         LLVMBuildStore(b, v_retval, v_resvaluep);
1387
1388                                         LLVMBuildBr(b, opblocks[i + 1]);
1389                                         break;
1390                                 }
1391
1392                         case EEOP_DISTINCT:
1393                         case EEOP_NOT_DISTINCT:
1394                                 {
1395                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
1396
1397                                         LLVMValueRef v_fcinfo;
1398                                         LLVMValueRef v_fcinfo_isnull;
1399
1400                                         LLVMValueRef v_argnull0,
1401                                                                 v_argisnull0;
1402                                         LLVMValueRef v_argnull1,
1403                                                                 v_argisnull1;
1404
1405                                         LLVMValueRef v_anyargisnull;
1406                                         LLVMValueRef v_bothargisnull;
1407
1408                                         LLVMValueRef v_result;
1409
1410                                         LLVMBasicBlockRef b_noargnull;
1411                                         LLVMBasicBlockRef b_checkbothargnull;
1412                                         LLVMBasicBlockRef b_bothargnull;
1413                                         LLVMBasicBlockRef b_anyargnull;
1414
1415                                         b_noargnull = l_bb_before_v(opblocks[i + 1], "op.%d.noargnull", i);
1416                                         b_checkbothargnull = l_bb_before_v(opblocks[i + 1], "op.%d.checkbothargnull", i);
1417                                         b_bothargnull = l_bb_before_v(opblocks[i + 1], "op.%d.bothargnull", i);
1418                                         b_anyargnull = l_bb_before_v(opblocks[i + 1], "op.%d.anyargnull", i);
1419
1420                                         v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
1421
1422                                         /* load args[0|1].isnull for both arguments */
1423                                         v_argnull0 = l_funcnull(b, v_fcinfo, 0);
1424                                         v_argisnull0 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull0,
1425                                                                                                  l_sbool_const(1), "");
1426                                         v_argnull1 = l_funcnull(b, v_fcinfo, 1);
1427                                         v_argisnull1 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull1,
1428                                                                                                  l_sbool_const(1), "");
1429
1430                                         v_anyargisnull = LLVMBuildOr(b, v_argisnull0, v_argisnull1, "");
1431                                         v_bothargisnull = LLVMBuildAnd(b, v_argisnull0, v_argisnull1, "");
1432
1433                                         /*
1434                                          * Check function arguments for NULLness: If either is
1435                                          * NULL, we check if both args are NULL. Otherwise call
1436                                          * comparator.
1437                                          */
1438                                         LLVMBuildCondBr(b, v_anyargisnull, b_checkbothargnull,
1439                                                                         b_noargnull);
1440
1441                                         /*
1442                                          * build block checking if any arg is null
1443                                          */
1444                                         LLVMPositionBuilderAtEnd(b, b_checkbothargnull);
1445                                         LLVMBuildCondBr(b, v_bothargisnull, b_bothargnull,
1446                                                                         b_anyargnull);
1447
1448
1449                                         /* Both NULL? Then is not distinct... */
1450                                         LLVMPositionBuilderAtEnd(b, b_bothargnull);
1451                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1452                                         if (opcode == EEOP_NOT_DISTINCT)
1453                                                 LLVMBuildStore(b, l_sizet_const(1), v_resvaluep);
1454                                         else
1455                                                 LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1456
1457                                         LLVMBuildBr(b, opblocks[i + 1]);
1458
1459                                         /* Only one is NULL? Then is distinct... */
1460                                         LLVMPositionBuilderAtEnd(b, b_anyargnull);
1461                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1462                                         if (opcode == EEOP_NOT_DISTINCT)
1463                                                 LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1464                                         else
1465                                                 LLVMBuildStore(b, l_sizet_const(1), v_resvaluep);
1466                                         LLVMBuildBr(b, opblocks[i + 1]);
1467
1468                                         /* neither argument is null: compare */
1469                                         LLVMPositionBuilderAtEnd(b, b_noargnull);
1470
1471                                         v_result = BuildV1Call(context, b, mod, fcinfo,
1472                                                                                    &v_fcinfo_isnull);
1473
1474                                         if (opcode == EEOP_DISTINCT)
1475                                         {
1476                                                 /* Must invert result of "=" */
1477                                                 v_result =
1478                                                         LLVMBuildZExt(b,
1479                                                                                   LLVMBuildICmp(b, LLVMIntEQ,
1480                                                                                                                 v_result,
1481                                                                                                                 l_sizet_const(0), ""),
1482                                                                                   TypeSizeT, "");
1483                                         }
1484
1485                                         LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp);
1486                                         LLVMBuildStore(b, v_result, v_resvaluep);
1487
1488                                         LLVMBuildBr(b, opblocks[i + 1]);
1489                                         break;
1490                                 }
1491
1492                         case EEOP_NULLIF:
1493                                 {
1494                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
1495
1496                                         LLVMValueRef v_fcinfo;
1497                                         LLVMValueRef v_fcinfo_isnull;
1498                                         LLVMValueRef v_argnull0;
1499                                         LLVMValueRef v_argnull1;
1500                                         LLVMValueRef v_anyargisnull;
1501                                         LLVMValueRef v_arg0;
1502                                         LLVMBasicBlockRef b_hasnull;
1503                                         LLVMBasicBlockRef b_nonull;
1504                                         LLVMBasicBlockRef b_argsequal;
1505                                         LLVMValueRef v_retval;
1506                                         LLVMValueRef v_argsequal;
1507
1508                                         b_hasnull = l_bb_before_v(opblocks[i + 1],
1509                                                                                           "b.%d.null-args", i);
1510                                         b_nonull = l_bb_before_v(opblocks[i + 1],
1511                                                                                          "b.%d.no-null-args", i);
1512                                         b_argsequal = l_bb_before_v(opblocks[i + 1],
1513                                                                                                 "b.%d.argsequal", i);
1514
1515                                         v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
1516
1517                                         /* if either argument is NULL they can't be equal */
1518                                         v_argnull0 = l_funcnull(b, v_fcinfo, 0);
1519                                         v_argnull1 = l_funcnull(b, v_fcinfo, 1);
1520
1521                                         v_anyargisnull =
1522                                                 LLVMBuildOr(b,
1523                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_argnull0,
1524                                                                                                   l_sbool_const(1), ""),
1525                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_argnull1,
1526                                                                                                   l_sbool_const(1), ""),
1527                                                                         "");
1528
1529                                         LLVMBuildCondBr(b, v_anyargisnull, b_hasnull, b_nonull);
1530
1531                                         /* one (or both) of the arguments are null, return arg[0] */
1532                                         LLVMPositionBuilderAtEnd(b, b_hasnull);
1533                                         v_arg0 = l_funcvalue(b, v_fcinfo, 0);
1534                                         LLVMBuildStore(b, v_argnull0, v_resnullp);
1535                                         LLVMBuildStore(b, v_arg0, v_resvaluep);
1536                                         LLVMBuildBr(b, opblocks[i + 1]);
1537
1538                                         /* build block to invoke function and check result */
1539                                         LLVMPositionBuilderAtEnd(b, b_nonull);
1540
1541                                         v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull);
1542
1543                                         /*
1544                                          * If result not null, and arguments are equal return null
1545                                          * (same result as if there'd been NULLs, hence reuse
1546                                          * b_hasnull).
1547                                          */
1548                                         v_argsequal = LLVMBuildAnd(b,
1549                                                                                            LLVMBuildICmp(b, LLVMIntEQ,
1550                                                                                                                          v_fcinfo_isnull,
1551                                                                                                                          l_sbool_const(0),
1552                                                                                                                          ""),
1553                                                                                            LLVMBuildICmp(b, LLVMIntEQ,
1554                                                                                                                          v_retval,
1555                                                                                                                          l_sizet_const(1),
1556                                                                                                                          ""),
1557                                                                                            "");
1558                                         LLVMBuildCondBr(b, v_argsequal, b_argsequal, b_hasnull);
1559
1560                                         /* build block setting result to NULL, if args are equal */
1561                                         LLVMPositionBuilderAtEnd(b, b_argsequal);
1562                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
1563                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1564                                         LLVMBuildStore(b, v_retval, v_resvaluep);
1565
1566                                         LLVMBuildBr(b, opblocks[i + 1]);
1567                                         break;
1568                                 }
1569
1570                         case EEOP_SQLVALUEFUNCTION:
1571                                 build_EvalXFunc(b, mod, "ExecEvalSQLValueFunction",
1572                                                                 v_state, v_econtext, op);
1573                                 LLVMBuildBr(b, opblocks[i + 1]);
1574                                 break;
1575
1576                         case EEOP_CURRENTOFEXPR:
1577                                 build_EvalXFunc(b, mod, "ExecEvalCurrentOfExpr",
1578                                                                 v_state, v_econtext, op);
1579                                 LLVMBuildBr(b, opblocks[i + 1]);
1580                                 break;
1581
1582                         case EEOP_NEXTVALUEEXPR:
1583                                 build_EvalXFunc(b, mod, "ExecEvalNextValueExpr",
1584                                                                 v_state, v_econtext, op);
1585                                 LLVMBuildBr(b, opblocks[i + 1]);
1586                                 break;
1587
1588                         case EEOP_ARRAYEXPR:
1589                                 build_EvalXFunc(b, mod, "ExecEvalArrayExpr",
1590                                                                 v_state, v_econtext, op);
1591                                 LLVMBuildBr(b, opblocks[i + 1]);
1592                                 break;
1593
1594                         case EEOP_ARRAYCOERCE:
1595                                 build_EvalXFunc(b, mod, "ExecEvalArrayCoerce",
1596                                                                 v_state, v_econtext, op);
1597                                 LLVMBuildBr(b, opblocks[i + 1]);
1598                                 break;
1599
1600                         case EEOP_ROW:
1601                                 build_EvalXFunc(b, mod, "ExecEvalRow",
1602                                                                 v_state, v_econtext, op);
1603                                 LLVMBuildBr(b, opblocks[i + 1]);
1604                                 break;
1605
1606                         case EEOP_ROWCOMPARE_STEP:
1607                                 {
1608                                         FunctionCallInfo fcinfo = op->d.rowcompare_step.fcinfo_data;
1609                                         LLVMValueRef v_fcinfo_isnull;
1610                                         LLVMBasicBlockRef b_null;
1611                                         LLVMBasicBlockRef b_compare;
1612                                         LLVMBasicBlockRef b_compare_result;
1613
1614                                         LLVMValueRef v_retval;
1615
1616                                         b_null = l_bb_before_v(opblocks[i + 1],
1617                                                                                    "op.%d.row-null", i);
1618                                         b_compare = l_bb_before_v(opblocks[i + 1],
1619                                                                                           "op.%d.row-compare", i);
1620                                         b_compare_result =
1621                                                 l_bb_before_v(opblocks[i + 1],
1622                                                                           "op.%d.row-compare-result",
1623                                                                           i);
1624
1625                                         /*
1626                                          * If function is strict, and either arg is null, we're
1627                                          * done.
1628                                          */
1629                                         if (op->d.rowcompare_step.finfo->fn_strict)
1630                                         {
1631                                                 LLVMValueRef v_fcinfo;
1632                                                 LLVMValueRef v_argnull0;
1633                                                 LLVMValueRef v_argnull1;
1634                                                 LLVMValueRef v_anyargisnull;
1635
1636                                                 v_fcinfo = l_ptr_const(fcinfo,
1637                                                                                            l_ptr(StructFunctionCallInfoData));
1638
1639                                                 v_argnull0 = l_funcnull(b, v_fcinfo, 0);
1640                                                 v_argnull1 = l_funcnull(b, v_fcinfo, 1);
1641
1642                                                 v_anyargisnull =
1643                                                         LLVMBuildOr(b,
1644                                                                                 LLVMBuildICmp(b,
1645                                                                                                           LLVMIntEQ,
1646                                                                                                           v_argnull0,
1647                                                                                                           l_sbool_const(1),
1648                                                                                                           ""),
1649                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
1650                                                                                                           v_argnull1,
1651                                                                                                           l_sbool_const(1), ""),
1652                                                                                 "");
1653
1654                                                 LLVMBuildCondBr(b, v_anyargisnull, b_null, b_compare);
1655                                         }
1656                                         else
1657                                         {
1658                                                 LLVMBuildBr(b, b_compare);
1659                                         }
1660
1661                                         /* build block invoking comparison function */
1662                                         LLVMPositionBuilderAtEnd(b, b_compare);
1663
1664                                         /* call function */
1665                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
1666                                                                                    &v_fcinfo_isnull);
1667                                         LLVMBuildStore(b, v_retval, v_resvaluep);
1668
1669                                         /* if result of function is NULL, force NULL result */
1670                                         LLVMBuildCondBr(b,
1671                                                                         LLVMBuildICmp(b,
1672                                                                                                   LLVMIntEQ,
1673                                                                                                   v_fcinfo_isnull,
1674                                                                                                   l_sbool_const(0),
1675                                                                                                   ""),
1676                                                                         b_compare_result,
1677                                                                         b_null);
1678
1679                                         /* build block analyzing the !NULL comparator result */
1680                                         LLVMPositionBuilderAtEnd(b, b_compare_result);
1681
1682                                         /* if results equal, compare next, otherwise done */
1683                                         LLVMBuildCondBr(b,
1684                                                                         LLVMBuildICmp(b,
1685                                                                                                   LLVMIntEQ,
1686                                                                                                   v_retval,
1687                                                                                                   l_sizet_const(0), ""),
1688                                                                         opblocks[i + 1],
1689                                                                         opblocks[op->d.rowcompare_step.jumpdone]);
1690
1691                                         /*
1692                                          * Build block handling NULL input or NULL comparator
1693                                          * result.
1694                                          */
1695                                         LLVMPositionBuilderAtEnd(b, b_null);
1696                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
1697                                         LLVMBuildBr(b, opblocks[op->d.rowcompare_step.jumpnull]);
1698
1699                                         break;
1700                                 }
1701
1702                         case EEOP_ROWCOMPARE_FINAL:
1703                                 {
1704                                         RowCompareType rctype = op->d.rowcompare_final.rctype;
1705
1706                                         LLVMValueRef v_cmpresult;
1707                                         LLVMValueRef v_result;
1708                                         LLVMIntPredicate predicate;
1709
1710                                         /*
1711                                          * Btree comparators return 32 bit results, need to be
1712                                          * careful about sign (used as a 64 bit value it's
1713                                          * otherwise wrong).
1714                                          */
1715                                         v_cmpresult =
1716                                                 LLVMBuildTrunc(b,
1717                                                                            LLVMBuildLoad(b, v_resvaluep, ""),
1718                                                                            LLVMInt32Type(), "");
1719
1720                                         switch (rctype)
1721                                         {
1722                                                 case ROWCOMPARE_LT:
1723                                                         predicate = LLVMIntSLT;
1724                                                         break;
1725                                                 case ROWCOMPARE_LE:
1726                                                         predicate = LLVMIntSLE;
1727                                                         break;
1728                                                 case ROWCOMPARE_GT:
1729                                                         predicate = LLVMIntSGT;
1730                                                         break;
1731                                                 case ROWCOMPARE_GE:
1732                                                         predicate = LLVMIntSGE;
1733                                                         break;
1734                                                 default:
1735                                                         /* EQ and NE cases aren't allowed here */
1736                                                         Assert(false);
1737                                                         predicate = 0;  /* prevent compiler warning */
1738                                                         break;
1739                                         }
1740
1741                                         v_result = LLVMBuildICmp(b,
1742                                                                                          predicate,
1743                                                                                          v_cmpresult,
1744                                                                                          l_int32_const(0),
1745                                                                                          "");
1746                                         v_result = LLVMBuildZExt(b, v_result, TypeSizeT, "");
1747
1748                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1749                                         LLVMBuildStore(b, v_result, v_resvaluep);
1750
1751                                         LLVMBuildBr(b, opblocks[i + 1]);
1752                                         break;
1753                                 }
1754
1755                         case EEOP_MINMAX:
1756                                 build_EvalXFunc(b, mod, "ExecEvalMinMax",
1757                                                                 v_state, v_econtext, op);
1758                                 LLVMBuildBr(b, opblocks[i + 1]);
1759                                 break;
1760
1761                         case EEOP_FIELDSELECT:
1762                                 build_EvalXFunc(b, mod, "ExecEvalFieldSelect",
1763                                                                 v_state, v_econtext, op);
1764                                 LLVMBuildBr(b, opblocks[i + 1]);
1765                                 break;
1766
1767                         case EEOP_FIELDSTORE_DEFORM:
1768                                 build_EvalXFunc(b, mod, "ExecEvalFieldStoreDeForm",
1769                                                                 v_state, v_econtext, op);
1770                                 LLVMBuildBr(b, opblocks[i + 1]);
1771                                 break;
1772
1773                         case EEOP_FIELDSTORE_FORM:
1774                                 build_EvalXFunc(b, mod, "ExecEvalFieldStoreForm",
1775                                                                 v_state, v_econtext, op);
1776                                 LLVMBuildBr(b, opblocks[i + 1]);
1777                                 break;
1778
1779                         case EEOP_ARRAYREF_SUBSCRIPT:
1780                                 {
1781                                         LLVMValueRef v_fn;
1782                                         int                     jumpdone = op->d.arrayref_subscript.jumpdone;
1783                                         LLVMValueRef v_params[2];
1784                                         LLVMValueRef v_ret;
1785
1786                                         v_fn = llvm_get_decl(mod, FuncExecEvalArrayRefSubscript);
1787
1788                                         v_params[0] = v_state;
1789                                         v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
1790                                         v_ret = LLVMBuildCall(b, v_fn,
1791                                                                                   v_params, lengthof(v_params), "");
1792                                         v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, "");
1793
1794                                         LLVMBuildCondBr(b,
1795                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_ret,
1796                                                                                                   l_sbool_const(1), ""),
1797                                                                         opblocks[i + 1],
1798                                                                         opblocks[jumpdone]);
1799                                         break;
1800                                 }
1801
1802                         case EEOP_DOMAIN_TESTVAL:
1803                                 {
1804                                         LLVMBasicBlockRef b_avail,
1805                                                                 b_notavail;
1806                                         LLVMValueRef v_casevaluep,
1807                                                                 v_casevalue;
1808                                         LLVMValueRef v_casenullp,
1809                                                                 v_casenull;
1810                                         LLVMValueRef v_casevaluenull;
1811
1812                                         b_avail = l_bb_before_v(opblocks[i + 1],
1813                                                                                         "op.%d.avail", i);
1814                                         b_notavail = l_bb_before_v(opblocks[i + 1],
1815                                                                                            "op.%d.notavail", i);
1816
1817                                         v_casevaluep = l_ptr_const(op->d.casetest.value,
1818                                                                                            l_ptr(TypeSizeT));
1819                                         v_casenullp = l_ptr_const(op->d.casetest.isnull,
1820                                                                                           l_ptr(TypeStorageBool));
1821
1822                                         v_casevaluenull =
1823                                                 LLVMBuildICmp(b, LLVMIntEQ,
1824                                                                           LLVMBuildPtrToInt(b, v_casevaluep,
1825                                                                                                                 TypeSizeT, ""),
1826                                                                           l_sizet_const(0), "");
1827                                         LLVMBuildCondBr(b,
1828                                                                         v_casevaluenull,
1829                                                                         b_notavail, b_avail);
1830
1831                                         /* if casetest != NULL */
1832                                         LLVMPositionBuilderAtEnd(b, b_avail);
1833                                         v_casevalue = LLVMBuildLoad(b, v_casevaluep, "");
1834                                         v_casenull = LLVMBuildLoad(b, v_casenullp, "");
1835                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1836                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1837                                         LLVMBuildBr(b, opblocks[i + 1]);
1838
1839                                         /* if casetest == NULL */
1840                                         LLVMPositionBuilderAtEnd(b, b_notavail);
1841                                         v_casevalue =
1842                                                 l_load_struct_gep(b, v_econtext,
1843                                                                                   FIELDNO_EXPRCONTEXT_DOMAINDATUM,
1844                                                                                   "");
1845                                         v_casenull =
1846                                                 l_load_struct_gep(b, v_econtext,
1847                                                                                   FIELDNO_EXPRCONTEXT_DOMAINNULL,
1848                                                                                   "");
1849                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1850                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1851
1852                                         LLVMBuildBr(b, opblocks[i + 1]);
1853                                         break;
1854                                 }
1855
1856                         case EEOP_DOMAIN_NOTNULL:
1857                                 build_EvalXFunc(b, mod, "ExecEvalConstraintNotNull",
1858                                                                 v_state, v_econtext, op);
1859                                 LLVMBuildBr(b, opblocks[i + 1]);
1860                                 break;
1861
1862                         case EEOP_DOMAIN_CHECK:
1863                                 build_EvalXFunc(b, mod, "ExecEvalConstraintCheck",
1864                                                                 v_state, v_econtext, op);
1865                                 LLVMBuildBr(b, opblocks[i + 1]);
1866                                 break;
1867
1868                         case EEOP_CONVERT_ROWTYPE:
1869                                 build_EvalXFunc(b, mod, "ExecEvalConvertRowtype",
1870                                                                 v_state, v_econtext, op);
1871                                 LLVMBuildBr(b, opblocks[i + 1]);
1872                                 break;
1873
1874                         case EEOP_SCALARARRAYOP:
1875                                 build_EvalXFunc(b, mod, "ExecEvalScalarArrayOp",
1876                                                                 v_state, v_econtext, op);
1877                                 LLVMBuildBr(b, opblocks[i + 1]);
1878                                 break;
1879
1880                         case EEOP_XMLEXPR:
1881                                 build_EvalXFunc(b, mod, "ExecEvalXmlExpr",
1882                                                                 v_state, v_econtext, op);
1883                                 LLVMBuildBr(b, opblocks[i + 1]);
1884                                 break;
1885
1886                         case EEOP_AGGREF:
1887                                 {
1888                                         AggrefExprState *aggref = op->d.aggref.astate;
1889                                         LLVMValueRef v_aggnop;
1890                                         LLVMValueRef v_aggno;
1891                                         LLVMValueRef value,
1892                                                                 isnull;
1893
1894                                         /*
1895                                          * At this point aggref->aggno is not yet set (it's set up
1896                                          * in ExecInitAgg() after initializing the expression). So
1897                                          * load it from memory each time round.
1898                                          */
1899                                         v_aggnop = l_ptr_const(&aggref->aggno,
1900                                                                                    l_ptr(LLVMInt32Type()));
1901                                         v_aggno = LLVMBuildLoad(b, v_aggnop, "v_aggno");
1902
1903                                         /* load agg value / null */
1904                                         value = l_load_gep1(b, v_aggvalues, v_aggno, "aggvalue");
1905                                         isnull = l_load_gep1(b, v_aggnulls, v_aggno, "aggnull");
1906
1907                                         /* and store result */
1908                                         LLVMBuildStore(b, value, v_resvaluep);
1909                                         LLVMBuildStore(b, isnull, v_resnullp);
1910
1911                                         LLVMBuildBr(b, opblocks[i + 1]);
1912                                         break;
1913                                 }
1914
1915                         case EEOP_GROUPING_FUNC:
1916                                 build_EvalXFunc(b, mod, "ExecEvalGroupingFunc",
1917                                                                 v_state, v_econtext, op);
1918                                 LLVMBuildBr(b, opblocks[i + 1]);
1919                                 break;
1920
1921                         case EEOP_WINDOW_FUNC:
1922                                 {
1923                                         WindowFuncExprState *wfunc = op->d.window_func.wfstate;
1924                                         LLVMValueRef v_wfuncnop;
1925                                         LLVMValueRef v_wfuncno;
1926                                         LLVMValueRef value,
1927                                                                 isnull;
1928
1929                                         /*
1930                                          * At this point aggref->wfuncno is not yet set (it's set
1931                                          * up in ExecInitWindowAgg() after initializing the
1932                                          * expression). So load it from memory each time round.
1933                                          */
1934                                         v_wfuncnop = l_ptr_const(&wfunc->wfuncno,
1935                                                                                          l_ptr(LLVMInt32Type()));
1936                                         v_wfuncno = LLVMBuildLoad(b, v_wfuncnop, "v_wfuncno");
1937
1938                                         /* load window func value / null */
1939                                         value = l_load_gep1(b, v_aggvalues, v_wfuncno,
1940                                                                                 "windowvalue");
1941                                         isnull = l_load_gep1(b, v_aggnulls, v_wfuncno,
1942                                                                                  "windownull");
1943
1944                                         LLVMBuildStore(b, value, v_resvaluep);
1945                                         LLVMBuildStore(b, isnull, v_resnullp);
1946
1947                                         LLVMBuildBr(b, opblocks[i + 1]);
1948                                         break;
1949                                 }
1950
1951                         case EEOP_SUBPLAN:
1952                                 build_EvalXFunc(b, mod, "ExecEvalSubPlan",
1953                                                                 v_state, v_econtext, op);
1954                                 LLVMBuildBr(b, opblocks[i + 1]);
1955                                 break;
1956
1957                         case EEOP_ALTERNATIVE_SUBPLAN:
1958                                 build_EvalXFunc(b, mod, "ExecEvalAlternativeSubPlan",
1959                                                                 v_state, v_econtext, op);
1960                                 LLVMBuildBr(b, opblocks[i + 1]);
1961                                 break;
1962
1963                         case EEOP_AGG_STRICT_DESERIALIZE:
1964                                 {
1965                                         FunctionCallInfo fcinfo = op->d.agg_deserialize.fcinfo_data;
1966                                         LLVMValueRef v_fcinfo;
1967                                         LLVMValueRef v_argnull0;
1968                                         LLVMBasicBlockRef b_deserialize;
1969
1970                                         b_deserialize = l_bb_before_v(opblocks[i + 1],
1971                                                                                                   "op.%d.deserialize", i);
1972
1973                                         v_fcinfo = l_ptr_const(fcinfo,
1974                                                                                    l_ptr(StructFunctionCallInfoData));
1975                                         v_argnull0 = l_funcnull(b, v_fcinfo, 0);
1976
1977                                         LLVMBuildCondBr(b,
1978                                                                         LLVMBuildICmp(b,
1979                                                                                                   LLVMIntEQ,
1980                                                                                                   v_argnull0,
1981                                                                                                   l_sbool_const(1),
1982                                                                                                   ""),
1983                                                                         opblocks[op->d.agg_deserialize.jumpnull],
1984                                                                         b_deserialize);
1985                                         LLVMPositionBuilderAtEnd(b, b_deserialize);
1986                                 }
1987                                 /* FALLTHROUGH */
1988
1989                         case EEOP_AGG_DESERIALIZE:
1990                                 {
1991                                         AggState   *aggstate;
1992                                         FunctionCallInfo fcinfo;
1993
1994                                         LLVMValueRef v_retval;
1995                                         LLVMValueRef v_fcinfo_isnull;
1996                                         LLVMValueRef v_tmpcontext;
1997                                         LLVMValueRef v_oldcontext;
1998
1999                                         aggstate = op->d.agg_deserialize.aggstate;
2000                                         fcinfo = op->d.agg_deserialize.fcinfo_data;
2001
2002                                         v_tmpcontext =
2003                                                 l_ptr_const(aggstate->tmpcontext->ecxt_per_tuple_memory,
2004                                                                         l_ptr(StructMemoryContextData));
2005                                         v_oldcontext = l_mcxt_switch(mod, b, v_tmpcontext);
2006                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
2007                                                                                    &v_fcinfo_isnull);
2008                                         l_mcxt_switch(mod, b, v_oldcontext);
2009
2010                                         LLVMBuildStore(b, v_retval, v_resvaluep);
2011                                         LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp);
2012
2013                                         LLVMBuildBr(b, opblocks[i + 1]);
2014                                         break;
2015                                 }
2016
2017                         case EEOP_AGG_STRICT_INPUT_CHECK_NULLS:
2018                         case EEOP_AGG_STRICT_INPUT_CHECK_ARGS:
2019                                 {
2020                                         int                     nargs = op->d.agg_strict_input_check.nargs;
2021                                         NullableDatum *args = op->d.agg_strict_input_check.args;
2022                                         bool       *nulls = op->d.agg_strict_input_check.nulls;
2023                                         int                     jumpnull;
2024                                         int                     argno;
2025
2026                                         LLVMValueRef v_argsp;
2027                                         LLVMValueRef v_nullsp;
2028                                         LLVMBasicBlockRef *b_checknulls;
2029
2030                                         Assert(nargs > 0);
2031
2032                                         jumpnull = op->d.agg_strict_input_check.jumpnull;
2033                                         v_argsp = l_ptr_const(args, l_ptr(StructNullableDatum));
2034                                         v_nullsp = l_ptr_const(nulls, l_ptr(TypeStorageBool));
2035
2036                                         /* create blocks for checking args */
2037                                         b_checknulls = palloc(sizeof(LLVMBasicBlockRef *) * nargs);
2038                                         for (argno = 0; argno < nargs; argno++)
2039                                         {
2040                                                 b_checknulls[argno] =
2041                                                         l_bb_before_v(opblocks[i + 1],
2042                                                                                   "op.%d.check-null.%d",
2043                                                                                   i, argno);
2044                                         }
2045
2046                                         LLVMBuildBr(b, b_checknulls[0]);
2047
2048                                         /* strict function, check for NULL args */
2049                                         for (argno = 0; argno < nargs; argno++)
2050                                         {
2051                                                 LLVMValueRef v_argno = l_int32_const(argno);
2052                                                 LLVMValueRef v_argisnull;
2053                                                 LLVMBasicBlockRef b_argnotnull;
2054
2055                                                 LLVMPositionBuilderAtEnd(b, b_checknulls[argno]);
2056
2057                                                 if (argno + 1 == nargs)
2058                                                         b_argnotnull = opblocks[i + 1];
2059                                                 else
2060                                                         b_argnotnull = b_checknulls[argno + 1];
2061
2062                                                 if (opcode == EEOP_AGG_STRICT_INPUT_CHECK_NULLS)
2063                                                         v_argisnull = l_load_gep1(b, v_nullsp, v_argno, "");
2064                                                 else
2065                                                 {
2066                                                         LLVMValueRef v_argn;
2067
2068                                                         v_argn = LLVMBuildGEP(b, v_argsp, &v_argno, 1, "");
2069                                                         v_argisnull =
2070                                                                 l_load_struct_gep(b, v_argn,
2071                                                                                                   FIELDNO_NULLABLE_DATUM_ISNULL,
2072                                                                                                   "");
2073                                                 }
2074
2075                                                 LLVMBuildCondBr(b,
2076                                                                                 LLVMBuildICmp(b,
2077                                                                                                           LLVMIntEQ,
2078                                                                                                           v_argisnull,
2079                                                                                                           l_sbool_const(1), ""),
2080                                                                                 opblocks[jumpnull],
2081                                                                                 b_argnotnull);
2082                                         }
2083
2084                                         break;
2085                                 }
2086
2087                         case EEOP_AGG_INIT_TRANS:
2088                                 {
2089                                         AggState   *aggstate;
2090                                         AggStatePerTrans pertrans;
2091
2092                                         LLVMValueRef v_aggstatep;
2093                                         LLVMValueRef v_pertransp;
2094
2095                                         LLVMValueRef v_allpergroupsp;
2096
2097                                         LLVMValueRef v_pergroupp;
2098
2099                                         LLVMValueRef v_setoff,
2100                                                                 v_transno;
2101
2102                                         LLVMValueRef v_notransvalue;
2103
2104                                         LLVMBasicBlockRef b_init;
2105
2106                                         aggstate = op->d.agg_init_trans.aggstate;
2107                                         pertrans = op->d.agg_init_trans.pertrans;
2108
2109                                         v_aggstatep = l_ptr_const(aggstate,
2110                                                                                           l_ptr(StructAggState));
2111                                         v_pertransp = l_ptr_const(pertrans,
2112                                                                                           l_ptr(StructAggStatePerTransData));
2113
2114                                         /*
2115                                          * pergroup = &aggstate->all_pergroups
2116                                          * [op->d.agg_init_trans_check.setoff]
2117                                          * [op->d.agg_init_trans_check.transno];
2118                                          */
2119                                         v_allpergroupsp =
2120                                                 l_load_struct_gep(b, v_aggstatep,
2121                                                                                   FIELDNO_AGGSTATE_ALL_PERGROUPS,
2122                                                                                   "aggstate.all_pergroups");
2123                                         v_setoff = l_int32_const(op->d.agg_init_trans.setoff);
2124                                         v_transno = l_int32_const(op->d.agg_init_trans.transno);
2125                                         v_pergroupp =
2126                                                 LLVMBuildGEP(b,
2127                                                                          l_load_gep1(b, v_allpergroupsp, v_setoff, ""),
2128                                                                          &v_transno, 1, "");
2129
2130                                         v_notransvalue =
2131                                                 l_load_struct_gep(b, v_pergroupp,
2132                                                                                   FIELDNO_AGGSTATEPERGROUPDATA_NOTRANSVALUE,
2133                                                                                   "notransvalue");
2134
2135                                         b_init = l_bb_before_v(opblocks[i + 1],
2136                                                                                    "op.%d.inittrans", i);
2137
2138                                         LLVMBuildCondBr(b,
2139                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_notransvalue,
2140                                                                                                   l_sbool_const(1), ""),
2141                                                                         b_init,
2142                                                                         opblocks[i + 1]);
2143
2144                                         LLVMPositionBuilderAtEnd(b, b_init);
2145
2146                                         {
2147                                                 LLVMValueRef params[3];
2148                                                 LLVMValueRef v_curaggcontext;
2149                                                 LLVMValueRef v_current_set;
2150                                                 LLVMValueRef v_aggcontext;
2151
2152                                                 v_aggcontext = l_ptr_const(op->d.agg_init_trans.aggcontext,
2153                                                                                                    l_ptr(StructExprContext));
2154
2155                                                 v_current_set =
2156                                                         LLVMBuildStructGEP(b,
2157                                                                                            v_aggstatep,
2158                                                                                            FIELDNO_AGGSTATE_CURRENT_SET,
2159                                                                                            "aggstate.current_set");
2160                                                 v_curaggcontext =
2161                                                         LLVMBuildStructGEP(b,
2162                                                                                            v_aggstatep,
2163                                                                                            FIELDNO_AGGSTATE_CURAGGCONTEXT,
2164                                                                                            "aggstate.curaggcontext");
2165
2166                                                 LLVMBuildStore(b, l_int32_const(op->d.agg_init_trans.setno),
2167                                                                            v_current_set);
2168                                                 LLVMBuildStore(b, v_aggcontext,
2169                                                                            v_curaggcontext);
2170
2171                                                 params[0] = v_aggstatep;
2172                                                 params[1] = v_pertransp;
2173                                                 params[2] = v_pergroupp;
2174
2175                                                 LLVMBuildCall(b,
2176                                                                           llvm_get_decl(mod, FuncExecAggInitGroup),
2177                                                                           params, lengthof(params),
2178                                                                           "");
2179                                         }
2180                                         LLVMBuildBr(b, opblocks[op->d.agg_init_trans.jumpnull]);
2181
2182                                         break;
2183                                 }
2184
2185                         case EEOP_AGG_STRICT_TRANS_CHECK:
2186                                 {
2187                                         AggState   *aggstate;
2188                                         LLVMValueRef v_setoff,
2189                                                                 v_transno;
2190
2191                                         LLVMValueRef v_aggstatep;
2192                                         LLVMValueRef v_allpergroupsp;
2193
2194                                         LLVMValueRef v_transnull;
2195                                         LLVMValueRef v_pergroupp;
2196
2197                                         int                     jumpnull = op->d.agg_strict_trans_check.jumpnull;
2198
2199                                         aggstate = op->d.agg_strict_trans_check.aggstate;
2200                                         v_aggstatep = l_ptr_const(aggstate, l_ptr(StructAggState));
2201
2202                                         /*
2203                                          * pergroup = &aggstate->all_pergroups
2204                                          * [op->d.agg_strict_trans_check.setoff]
2205                                          * [op->d.agg_init_trans_check.transno];
2206                                          */
2207                                         v_allpergroupsp =
2208                                                 l_load_struct_gep(b, v_aggstatep,
2209                                                                                   FIELDNO_AGGSTATE_ALL_PERGROUPS,
2210                                                                                   "aggstate.all_pergroups");
2211                                         v_setoff =
2212                                                 l_int32_const(op->d.agg_strict_trans_check.setoff);
2213                                         v_transno =
2214                                                 l_int32_const(op->d.agg_strict_trans_check.transno);
2215                                         v_pergroupp =
2216                                                 LLVMBuildGEP(b,
2217                                                                          l_load_gep1(b, v_allpergroupsp, v_setoff, ""),
2218                                                                          &v_transno, 1, "");
2219
2220                                         v_transnull =
2221                                                 l_load_struct_gep(b, v_pergroupp,
2222                                                                                   FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL,
2223                                                                                   "transnull");
2224
2225                                         LLVMBuildCondBr(b,
2226                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_transnull,
2227                                                                                                   l_sbool_const(1), ""),
2228                                                                         opblocks[jumpnull],
2229                                                                         opblocks[i + 1]);
2230
2231                                         break;
2232                                 }
2233
2234                         case EEOP_AGG_PLAIN_TRANS_BYVAL:
2235                         case EEOP_AGG_PLAIN_TRANS:
2236                                 {
2237                                         AggState   *aggstate;
2238                                         AggStatePerTrans pertrans;
2239                                         FunctionCallInfo fcinfo;
2240
2241                                         LLVMValueRef v_aggstatep;
2242                                         LLVMValueRef v_fcinfo;
2243                                         LLVMValueRef v_fcinfo_isnull;
2244
2245                                         LLVMValueRef v_transvaluep;
2246                                         LLVMValueRef v_transnullp;
2247
2248                                         LLVMValueRef v_setoff;
2249                                         LLVMValueRef v_transno;
2250
2251                                         LLVMValueRef v_aggcontext;
2252
2253                                         LLVMValueRef v_allpergroupsp;
2254                                         LLVMValueRef v_current_setp;
2255                                         LLVMValueRef v_current_pertransp;
2256                                         LLVMValueRef v_curaggcontext;
2257
2258                                         LLVMValueRef v_pertransp;
2259
2260                                         LLVMValueRef v_pergroupp;
2261
2262                                         LLVMValueRef v_retval;
2263
2264                                         LLVMValueRef v_tmpcontext;
2265                                         LLVMValueRef v_oldcontext;
2266
2267                                         aggstate = op->d.agg_trans.aggstate;
2268                                         pertrans = op->d.agg_trans.pertrans;
2269
2270                                         fcinfo = pertrans->transfn_fcinfo;
2271
2272                                         v_aggstatep = l_ptr_const(aggstate,
2273                                                                                           l_ptr(StructAggState));
2274                                         v_pertransp = l_ptr_const(pertrans,
2275                                                                                           l_ptr(StructAggStatePerTransData));
2276
2277                                         /*
2278                                          * pergroup = &aggstate->all_pergroups
2279                                          * [op->d.agg_strict_trans_check.setoff]
2280                                          * [op->d.agg_init_trans_check.transno];
2281                                          */
2282                                         v_allpergroupsp =
2283                                                 l_load_struct_gep(b, v_aggstatep,
2284                                                                                   FIELDNO_AGGSTATE_ALL_PERGROUPS,
2285                                                                                   "aggstate.all_pergroups");
2286                                         v_setoff = l_int32_const(op->d.agg_trans.setoff);
2287                                         v_transno = l_int32_const(op->d.agg_trans.transno);
2288                                         v_pergroupp =
2289                                                 LLVMBuildGEP(b,
2290                                                                          l_load_gep1(b, v_allpergroupsp, v_setoff, ""),
2291                                                                          &v_transno, 1, "");
2292
2293                                         v_fcinfo = l_ptr_const(fcinfo,
2294                                                                                    l_ptr(StructFunctionCallInfoData));
2295                                         v_aggcontext = l_ptr_const(op->d.agg_trans.aggcontext,
2296                                                                                            l_ptr(StructExprContext));
2297
2298                                         v_current_setp =
2299                                                 LLVMBuildStructGEP(b,
2300                                                                                    v_aggstatep,
2301                                                                                    FIELDNO_AGGSTATE_CURRENT_SET,
2302                                                                                    "aggstate.current_set");
2303                                         v_curaggcontext =
2304                                                 LLVMBuildStructGEP(b,
2305                                                                                    v_aggstatep,
2306                                                                                    FIELDNO_AGGSTATE_CURAGGCONTEXT,
2307                                                                                    "aggstate.curaggcontext");
2308                                         v_current_pertransp =
2309                                                 LLVMBuildStructGEP(b,
2310                                                                                    v_aggstatep,
2311                                                                                    FIELDNO_AGGSTATE_CURPERTRANS,
2312                                                                                    "aggstate.curpertrans");
2313
2314                                         /* set aggstate globals */
2315                                         LLVMBuildStore(b, v_aggcontext, v_curaggcontext);
2316                                         LLVMBuildStore(b, l_int32_const(op->d.agg_trans.setno),
2317                                                                    v_current_setp);
2318                                         LLVMBuildStore(b, v_pertransp, v_current_pertransp);
2319
2320                                         /* invoke transition function in per-tuple context */
2321                                         v_tmpcontext =
2322                                                 l_ptr_const(aggstate->tmpcontext->ecxt_per_tuple_memory,
2323                                                                         l_ptr(StructMemoryContextData));
2324                                         v_oldcontext = l_mcxt_switch(mod, b, v_tmpcontext);
2325
2326                                         /* store transvalue in fcinfo->args[0] */
2327                                         v_transvaluep =
2328                                                 LLVMBuildStructGEP(b, v_pergroupp,
2329                                                                                    FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUE,
2330                                                                                    "transvalue");
2331                                         v_transnullp =
2332                                                 LLVMBuildStructGEP(b, v_pergroupp,
2333                                                                                    FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL,
2334                                                                                    "transnullp");
2335                                         LLVMBuildStore(b,
2336                                                                    LLVMBuildLoad(b, v_transvaluep,
2337                                                                                                  "transvalue"),
2338                                                                    l_funcvaluep(b, v_fcinfo, 0));
2339                                         LLVMBuildStore(b,
2340                                                                    LLVMBuildLoad(b, v_transnullp, "transnull"),
2341                                                                    l_funcnullp(b, v_fcinfo, 0));
2342
2343                                         /* and invoke transition function */
2344                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
2345                                                                                    &v_fcinfo_isnull);
2346
2347                                         /*
2348                                          * For pass-by-ref datatype, must copy the new value into
2349                                          * aggcontext and free the prior transValue.  But if
2350                                          * transfn returned a pointer to its first input, we don't
2351                                          * need to do anything.  Also, if transfn returned a
2352                                          * pointer to a R/W expanded object that is already a
2353                                          * child of the aggcontext, assume we can adopt that value
2354                                          * without copying it.
2355                                          */
2356                                         if (opcode == EEOP_AGG_PLAIN_TRANS)
2357                                         {
2358                                                 LLVMBasicBlockRef b_call;
2359                                                 LLVMBasicBlockRef b_nocall;
2360                                                 LLVMValueRef v_fn;
2361                                                 LLVMValueRef v_transvalue;
2362                                                 LLVMValueRef v_transnull;
2363                                                 LLVMValueRef v_newval;
2364                                                 LLVMValueRef params[6];
2365
2366                                                 b_call = l_bb_before_v(opblocks[i + 1],
2367                                                                                            "op.%d.transcall", i);
2368                                                 b_nocall = l_bb_before_v(opblocks[i + 1],
2369                                                                                                  "op.%d.transnocall", i);
2370
2371                                                 v_transvalue = LLVMBuildLoad(b, v_transvaluep, "");
2372                                                 v_transnull = LLVMBuildLoad(b, v_transnullp, "");
2373
2374                                                 /*
2375                                                  * DatumGetPointer(newVal) !=
2376                                                  * DatumGetPointer(pergroup->transValue))
2377                                                  */
2378                                                 LLVMBuildCondBr(b,
2379                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
2380                                                                                                           v_transvalue,
2381                                                                                                           v_retval, ""),
2382                                                                                 b_nocall, b_call);
2383
2384                                                 /* returned datum not passed datum, reparent */
2385                                                 LLVMPositionBuilderAtEnd(b, b_call);
2386
2387                                                 params[0] = v_aggstatep;
2388                                                 params[1] = v_pertransp;
2389                                                 params[2] = v_retval;
2390                                                 params[3] = LLVMBuildTrunc(b, v_fcinfo_isnull,
2391                                                                                                    TypeParamBool, "");
2392                                                 params[4] = v_transvalue;
2393                                                 params[5] = LLVMBuildTrunc(b, v_transnull,
2394                                                                                                    TypeParamBool, "");
2395
2396                                                 v_fn = llvm_get_decl(mod, FuncExecAggTransReparent);
2397                                                 v_newval =
2398                                                         LLVMBuildCall(b, v_fn,
2399                                                                                   params, lengthof(params),
2400                                                                                   "");
2401
2402                                                 /* store trans value */
2403                                                 LLVMBuildStore(b, v_newval, v_transvaluep);
2404                                                 LLVMBuildStore(b, v_fcinfo_isnull, v_transnullp);
2405
2406                                                 l_mcxt_switch(mod, b, v_oldcontext);
2407                                                 LLVMBuildBr(b, opblocks[i + 1]);
2408
2409                                                 /* returned datum passed datum, no need to reparent */
2410                                                 LLVMPositionBuilderAtEnd(b, b_nocall);
2411                                         }
2412
2413                                         /* store trans value */
2414                                         LLVMBuildStore(b, v_retval, v_transvaluep);
2415                                         LLVMBuildStore(b, v_fcinfo_isnull, v_transnullp);
2416
2417                                         l_mcxt_switch(mod, b, v_oldcontext);
2418
2419                                         LLVMBuildBr(b, opblocks[i + 1]);
2420                                         break;
2421                                 }
2422
2423                         case EEOP_AGG_ORDERED_TRANS_DATUM:
2424                                 build_EvalXFunc(b, mod, "ExecEvalAggOrderedTransDatum",
2425                                                                 v_state, v_econtext, op);
2426                                 LLVMBuildBr(b, opblocks[i + 1]);
2427                                 break;
2428
2429                         case EEOP_AGG_ORDERED_TRANS_TUPLE:
2430                                 build_EvalXFunc(b, mod, "ExecEvalAggOrderedTransTuple",
2431                                                                 v_state, v_econtext, op);
2432                                 LLVMBuildBr(b, opblocks[i + 1]);
2433                                 break;
2434
2435                         case EEOP_LAST:
2436                                 Assert(false);
2437                                 break;
2438                 }
2439         }
2440
2441         LLVMDisposeBuilder(b);
2442
2443         /*
2444          * Don't immediately emit function, instead do so the first time the
2445          * expression is actually evaluated. That allows to emit a lot of
2446          * functions together, avoiding a lot of repeated llvm and memory
2447          * remapping overhead.
2448          */
2449         {
2450
2451                 CompiledExprState *cstate = palloc0(sizeof(CompiledExprState));
2452
2453                 cstate->context = context;
2454                 cstate->funcname = funcname;
2455
2456                 state->evalfunc = ExecRunCompiledExpr;
2457                 state->evalfunc_private = cstate;
2458         }
2459
2460         llvm_leave_fatal_on_oom();
2461
2462         INSTR_TIME_SET_CURRENT(endtime);
2463         INSTR_TIME_ACCUM_DIFF(context->base.instr.generation_counter,
2464                                                   endtime, starttime);
2465
2466         return true;
2467 }
2468
2469 /*
2470  * Run compiled expression.
2471  *
2472  * This will only be called the first time a JITed expression is called. We
2473  * first make sure the expression is still up2date, and then get a pointer to
2474  * the emitted function. The latter can be the first thing that triggers
2475  * optimizing and emitting all the generated functions.
2476  */
2477 static Datum
2478 ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull)
2479 {
2480         CompiledExprState *cstate = state->evalfunc_private;
2481         ExprStateEvalFunc func;
2482
2483         CheckExprStillValid(state, econtext);
2484
2485         llvm_enter_fatal_on_oom();
2486         func = (ExprStateEvalFunc) llvm_get_function(cstate->context,
2487                                                                                                  cstate->funcname);
2488         llvm_leave_fatal_on_oom();
2489         Assert(func);
2490
2491         /* remove indirection via this function for future calls */
2492         state->evalfunc = func;
2493
2494         return func(state, econtext, isNull);
2495 }
2496
2497 static LLVMValueRef
2498 BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b,
2499                         LLVMModuleRef mod, FunctionCallInfo fcinfo,
2500                         LLVMValueRef *v_fcinfo_isnull)
2501 {
2502         LLVMValueRef v_fn;
2503         LLVMValueRef v_fcinfo_isnullp;
2504         LLVMValueRef v_retval;
2505         LLVMValueRef v_fcinfo;
2506
2507         v_fn = llvm_function_reference(context, b, mod, fcinfo);
2508
2509         v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
2510         v_fcinfo_isnullp = LLVMBuildStructGEP(b, v_fcinfo,
2511                                                                                   FIELDNO_FUNCTIONCALLINFODATA_ISNULL,
2512                                                                                   "v_fcinfo_isnull");
2513         LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_isnullp);
2514
2515         v_retval = LLVMBuildCall(b, v_fn, &v_fcinfo, 1, "funccall");
2516
2517         if (v_fcinfo_isnull)
2518                 *v_fcinfo_isnull = LLVMBuildLoad(b, v_fcinfo_isnullp, "");
2519
2520         /*
2521          * Add lifetime-end annotation, signalling that writes to memory don't
2522          * have to be retained (important for inlining potential).
2523          */
2524         {
2525                 LLVMValueRef v_lifetime = create_LifetimeEnd(mod);
2526                 LLVMValueRef params[2];
2527
2528                 params[0] = l_int64_const(sizeof(NullableDatum) * fcinfo->nargs);
2529                 params[1] = l_ptr_const(fcinfo->args, l_ptr(LLVMInt8Type()));
2530                 LLVMBuildCall(b, v_lifetime, params, lengthof(params), "");
2531
2532                 params[0] = l_int64_const(sizeof(fcinfo->isnull));
2533                 params[1] = l_ptr_const(&fcinfo->isnull, l_ptr(LLVMInt8Type()));
2534                 LLVMBuildCall(b, v_lifetime, params, lengthof(params), "");
2535         }
2536
2537         return v_retval;
2538 }
2539
2540 /*
2541  * Implement an expression step by calling the function funcname.
2542  */
2543 static void
2544 build_EvalXFunc(LLVMBuilderRef b, LLVMModuleRef mod, const char *funcname,
2545                                 LLVMValueRef v_state, LLVMValueRef v_econtext,
2546                                 ExprEvalStep *op)
2547 {
2548         LLVMTypeRef sig;
2549         LLVMValueRef v_fn;
2550         LLVMTypeRef param_types[3];
2551         LLVMValueRef params[3];
2552
2553         v_fn = LLVMGetNamedFunction(mod, funcname);
2554         if (!v_fn)
2555         {
2556                 param_types[0] = l_ptr(StructExprState);
2557                 param_types[1] = l_ptr(StructExprEvalStep);
2558                 param_types[2] = l_ptr(StructExprContext);
2559
2560                 sig = LLVMFunctionType(LLVMVoidType(),
2561                                                            param_types, lengthof(param_types),
2562                                                            false);
2563                 v_fn = LLVMAddFunction(mod, funcname, sig);
2564         }
2565
2566         params[0] = v_state;
2567         params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
2568         params[2] = v_econtext;
2569
2570         LLVMBuildCall(b,
2571                                   v_fn,
2572                                   params, lengthof(params), "");
2573 }
2574
2575 static LLVMValueRef
2576 create_LifetimeEnd(LLVMModuleRef mod)
2577 {
2578         LLVMTypeRef sig;
2579         LLVMValueRef fn;
2580         LLVMTypeRef param_types[2];
2581
2582         /* LLVM 5+ has a variadic pointer argument */
2583 #if LLVM_VERSION_MAJOR < 5
2584         const char *nm = "llvm.lifetime.end";
2585 #else
2586         const char *nm = "llvm.lifetime.end.p0i8";
2587 #endif
2588
2589         fn = LLVMGetNamedFunction(mod, nm);
2590         if (fn)
2591                 return fn;
2592
2593         param_types[0] = LLVMInt64Type();
2594         param_types[1] = l_ptr(LLVMInt8Type());
2595
2596         sig = LLVMFunctionType(LLVMVoidType(),
2597                                                    param_types, lengthof(param_types),
2598                                                    false);
2599         fn = LLVMAddFunction(mod, nm, sig);
2600
2601         LLVMSetFunctionCallConv(fn, LLVMCCallConv);
2602
2603         Assert(LLVMGetIntrinsicID(fn));
2604
2605         return fn;
2606 }