granicus.if.org Git - postgresql/blob - src/backend/jit/llvm/llvmjit_expr.c

   1 /*-------------------------------------------------------------------------
   2  *
   3  * llvmjit_expr.c
   4  *        JIT compile expressions.
   5  *
   6  * Portions Copyright (c) 1996-2017, PostgreSQL Global Development Group
   7  * Portions Copyright (c) 1994, Regents of the University of California
   8  *
   9  *
  10  * IDENTIFICATION
  11  *        src/backend/jit/llvm/llvmjit_expr.c
  12  *
  13  *-------------------------------------------------------------------------
  14  */
  15
  16 #include "postgres.h"
  17
  18 #include <llvm-c/Core.h>
  19 #include <llvm-c/Target.h>
  20
  21 #include "access/htup_details.h"
  22 #include "access/nbtree.h"
  23 #include "access/tupconvert.h"
  24 #include "catalog/objectaccess.h"
  25 #include "catalog/pg_type.h"
  26 #include "executor/execdebug.h"
  27 #include "executor/nodeAgg.h"
  28 #include "executor/nodeSubplan.h"
  29 #include "executor/execExpr.h"
  30 #include "funcapi.h"
  31 #include "jit/llvmjit.h"
  32 #include "jit/llvmjit_emit.h"
  33 #include "miscadmin.h"
  34 #include "nodes/makefuncs.h"
  35 #include "nodes/nodeFuncs.h"
  36 #include "optimizer/planner.h"
  37 #include "parser/parse_coerce.h"
  38 #include "parser/parsetree.h"
  39 #include "pgstat.h"
  40 #include "utils/acl.h"
  41 #include "utils/builtins.h"
  42 #include "utils/date.h"
  43 #include "utils/fmgrtab.h"
  44 #include "utils/lsyscache.h"
  45 #include "utils/memutils.h"
  46 #include "utils/timestamp.h"
  47 #include "utils/typcache.h"
  48 #include "utils/xml.h"
  49
  50
  51 typedef struct CompiledExprState
  52 {
  53         LLVMJitContext *context;
  54         const char *funcname;
  55 } CompiledExprState;
  56
  57
  58 static Datum ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull);
  59
  60 static LLVMValueRef BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b,
  61                         LLVMModuleRef mod, FunctionCallInfo fcinfo,
  62                         LLVMValueRef *v_fcinfo_isnull);
  63 static void build_EvalXFunc(LLVMBuilderRef b, LLVMModuleRef mod,
  64                                 const char *funcname,
  65                                 LLVMValueRef v_state, LLVMValueRef v_econtext,
  66                                 ExprEvalStep *op);
  67 static LLVMValueRef create_LifetimeEnd(LLVMModuleRef mod);
  68
  69
  70 /*
  71  * JIT compile expression.
  72  */
  73 bool
  74 llvm_compile_expr(ExprState *state)
  75 {
  76         PlanState  *parent = state->parent;
  77         int                     i;
  78         char       *funcname;
  79
  80         LLVMJitContext *context = NULL;
  81
  82         LLVMBuilderRef b;
  83         LLVMModuleRef mod;
  84         LLVMTypeRef eval_sig;
  85         LLVMValueRef eval_fn;
  86         LLVMBasicBlockRef entry;
  87         LLVMBasicBlockRef *opblocks;
  88
  89         /* state itself */
  90         LLVMValueRef v_state;
  91         LLVMValueRef v_econtext;
  92
  93         /* returnvalue */
  94         LLVMValueRef v_isnullp;
  95
  96         /* tmp vars in state */
  97         LLVMValueRef v_tmpvaluep;
  98         LLVMValueRef v_tmpisnullp;
  99
 100         /* slots */
 101         LLVMValueRef v_innerslot;
 102         LLVMValueRef v_outerslot;
 103         LLVMValueRef v_scanslot;
 104         LLVMValueRef v_resultslot;
 105
 106         /* nulls/values of slots */
 107         LLVMValueRef v_innervalues;
 108         LLVMValueRef v_innernulls;
 109         LLVMValueRef v_outervalues;
 110         LLVMValueRef v_outernulls;
 111         LLVMValueRef v_scanvalues;
 112         LLVMValueRef v_scannulls;
 113         LLVMValueRef v_resultvalues;
 114         LLVMValueRef v_resultnulls;
 115
 116         /* stuff in econtext */
 117         LLVMValueRef v_aggvalues;
 118         LLVMValueRef v_aggnulls;
 119
 120         instr_time      starttime;
 121         instr_time      endtime;
 122
 123         llvm_enter_fatal_on_oom();
 124
 125         /* get or create JIT context */
 126         if (parent && parent->state->es_jit)
 127         {
 128                 context = (LLVMJitContext *) parent->state->es_jit;
 129         }
 130         else
 131         {
 132                 context = llvm_create_context(parent->state->es_jit_flags);
 133
 134                 if (parent)
 135                 {
 136                         parent->state->es_jit = &context->base;
 137                 }
 138
 139         }
 140
 141         INSTR_TIME_SET_CURRENT(starttime);
 142
 143         mod = llvm_mutable_module(context);
 144
 145         b = LLVMCreateBuilder();
 146
 147         funcname = llvm_expand_funcname(context, "evalexpr");
 148
 149         /* Create the signature and function */
 150         {
 151                 LLVMTypeRef param_types[3];
 152
 153                 param_types[0] = l_ptr(StructExprState);        /* state */
 154                 param_types[1] = l_ptr(StructExprContext);      /* econtext */
 155                 param_types[2] = l_ptr(TypeParamBool);  /* isnull */
 156
 157                 eval_sig = LLVMFunctionType(TypeSizeT,
 158                                                                         param_types, lengthof(param_types),
 159                                                                         false);
 160         }
 161         eval_fn = LLVMAddFunction(mod, funcname, eval_sig);
 162         LLVMSetLinkage(eval_fn, LLVMExternalLinkage);
 163         LLVMSetVisibility(eval_fn, LLVMDefaultVisibility);
 164         llvm_copy_attributes(AttributeTemplate, eval_fn);
 165
 166         entry = LLVMAppendBasicBlock(eval_fn, "entry");
 167
 168         /* build state */
 169         v_state = LLVMGetParam(eval_fn, 0);
 170         v_econtext = LLVMGetParam(eval_fn, 1);
 171         v_isnullp = LLVMGetParam(eval_fn, 2);
 172
 173         LLVMPositionBuilderAtEnd(b, entry);
 174
 175         v_tmpvaluep = LLVMBuildStructGEP(b, v_state,
 176                                                                          FIELDNO_EXPRSTATE_RESVALUE,
 177                                                                          "v.state.resvalue");
 178         v_tmpisnullp = LLVMBuildStructGEP(b, v_state,
 179                                                                           FIELDNO_EXPRSTATE_RESNULL,
 180                                                                           "v.state.resnull");
 181
 182         /* build global slots */
 183         v_scanslot = l_load_struct_gep(b, v_econtext,
 184                                                                    FIELDNO_EXPRCONTEXT_SCANTUPLE,
 185                                                                    "v_scanslot");
 186         v_innerslot = l_load_struct_gep(b, v_econtext,
 187                                                                         FIELDNO_EXPRCONTEXT_INNERTUPLE,
 188                                                                         "v_innerslot");
 189         v_outerslot = l_load_struct_gep(b, v_econtext,
 190                                                                         FIELDNO_EXPRCONTEXT_OUTERTUPLE,
 191                                                                         "v_outerslot");
 192         v_resultslot = l_load_struct_gep(b, v_state,
 193                                                                          FIELDNO_EXPRSTATE_RESULTSLOT,
 194                                                                          "v_resultslot");
 195
 196         /* build global values/isnull pointers */
 197         v_scanvalues = l_load_struct_gep(b, v_scanslot,
 198                                                                          FIELDNO_TUPLETABLESLOT_VALUES,
 199                                                                          "v_scanvalues");
 200         v_scannulls = l_load_struct_gep(b, v_scanslot,
 201                                                                         FIELDNO_TUPLETABLESLOT_ISNULL,
 202                                                                         "v_scannulls");
 203         v_innervalues = l_load_struct_gep(b, v_innerslot,
 204                                                                           FIELDNO_TUPLETABLESLOT_VALUES,
 205                                                                           "v_innervalues");
 206         v_innernulls = l_load_struct_gep(b, v_innerslot,
 207                                                                          FIELDNO_TUPLETABLESLOT_ISNULL,
 208                                                                          "v_innernulls");
 209         v_outervalues = l_load_struct_gep(b, v_outerslot,
 210                                                                           FIELDNO_TUPLETABLESLOT_VALUES,
 211                                                                           "v_outervalues");
 212         v_outernulls = l_load_struct_gep(b, v_outerslot,
 213                                                                          FIELDNO_TUPLETABLESLOT_ISNULL,
 214                                                                          "v_outernulls");
 215         v_resultvalues = l_load_struct_gep(b, v_resultslot,
 216                                                                            FIELDNO_TUPLETABLESLOT_VALUES,
 217                                                                            "v_resultvalues");
 218         v_resultnulls = l_load_struct_gep(b, v_resultslot,
 219                                                                           FIELDNO_TUPLETABLESLOT_ISNULL,
 220                                                                           "v_resultnulls");
 221
 222         /* aggvalues/aggnulls */
 223         v_aggvalues = l_load_struct_gep(b, v_econtext,
 224                                                                         FIELDNO_EXPRCONTEXT_AGGVALUES,
 225                                                                         "v.econtext.aggvalues");
 226         v_aggnulls = l_load_struct_gep(b, v_econtext,
 227                                                                    FIELDNO_EXPRCONTEXT_AGGNULLS,
 228                                                                    "v.econtext.aggnulls");
 229
 230         /* allocate blocks for each op upfront, so we can do jumps easily */
 231         opblocks = palloc(sizeof(LLVMBasicBlockRef) * state->steps_len);
 232         for (i = 0; i < state->steps_len; i++)
 233                 opblocks[i] = l_bb_append_v(eval_fn, "b.op.%d.start", i);
 234
 235         /* jump from entry to first block */
 236         LLVMBuildBr(b, opblocks[0]);
 237
 238         for (i = 0; i < state->steps_len; i++)
 239         {
 240                 ExprEvalStep *op;
 241                 ExprEvalOp      opcode;
 242                 LLVMValueRef v_resvaluep;
 243                 LLVMValueRef v_resnullp;
 244
 245                 LLVMPositionBuilderAtEnd(b, opblocks[i]);
 246
 247                 op = &state->steps[i];
 248                 opcode = ExecEvalStepOp(state, op);
 249
 250                 v_resvaluep = l_ptr_const(op->resvalue, l_ptr(TypeSizeT));
 251                 v_resnullp = l_ptr_const(op->resnull, l_ptr(TypeStorageBool));
 252
 253                 switch (opcode)
 254                 {
 255                         case EEOP_DONE:
 256                                 {
 257                                         LLVMValueRef v_tmpisnull,
 258                                                                 v_tmpvalue;
 259
 260                                         v_tmpvalue = LLVMBuildLoad(b, v_tmpvaluep, "");
 261                                         v_tmpisnull = LLVMBuildLoad(b, v_tmpisnullp, "");
 262                                         v_tmpisnull =
 263                                                 LLVMBuildTrunc(b, v_tmpisnull, TypeParamBool, "");
 264
 265                                         LLVMBuildStore(b, v_tmpisnull, v_isnullp);
 266
 267                                         LLVMBuildRet(b, v_tmpvalue);
 268                                         break;
 269                                 }
 270
 271                         case EEOP_INNER_FETCHSOME:
 272                         case EEOP_OUTER_FETCHSOME:
 273                         case EEOP_SCAN_FETCHSOME:
 274                                 {
 275                                         TupleDesc       desc = NULL;
 276                                         LLVMValueRef v_slot;
 277                                         LLVMBasicBlockRef b_fetch;
 278                                         LLVMValueRef v_nvalid;
 279
 280                                         b_fetch = l_bb_before_v(opblocks[i + 1],
 281                                                                                         "op.%d.fetch", i);
 282
 283                                         if (op->d.fetch.known_desc)
 284                                                 desc = op->d.fetch.known_desc;
 285
 286                                         if (opcode == EEOP_INNER_FETCHSOME)
 287                                         {
 288                                                 PlanState  *is = innerPlanState(parent);
 289
 290                                                 v_slot = v_innerslot;
 291
 292                                                 if (!desc &&
 293                                                         is &&
 294                                                         is->ps_ResultTupleSlot &&
 295                                                         is->ps_ResultTupleSlot->tts_fixedTupleDescriptor)
 296                                                         desc = is->ps_ResultTupleSlot->tts_tupleDescriptor;
 297                                         }
 298                                         else if (opcode == EEOP_OUTER_FETCHSOME)
 299                                         {
 300                                                 PlanState  *os = outerPlanState(parent);
 301
 302                                                 v_slot = v_outerslot;
 303
 304                                                 if (!desc &&
 305                                                         os &&
 306                                                         os->ps_ResultTupleSlot &&
 307                                                         os->ps_ResultTupleSlot->tts_fixedTupleDescriptor)
 308                                                         desc = os->ps_ResultTupleSlot->tts_tupleDescriptor;
 309                                         }
 310                                         else
 311                                         {
 312                                                 v_slot = v_scanslot;
 313                                                 if (!desc && parent)
 314                                                         desc = parent->scandesc;
 315                                         }
 316
 317                                         /*
 318                                          * Check if all required attributes are available, or
 319                                          * whether deforming is required.
 320                                          */
 321                                         v_nvalid =
 322                                                 l_load_struct_gep(b, v_slot,
 323                                                                                   FIELDNO_TUPLETABLESLOT_NVALID,
 324                                                                                   "");
 325                                         LLVMBuildCondBr(b,
 326                                                                         LLVMBuildICmp(b, LLVMIntUGE, v_nvalid,
 327                                                                                                   l_int32_const(op->d.fetch.last_var),
 328                                                                                                   ""),
 329                                                                         opblocks[i + 1], b_fetch);
 330
 331                                         LLVMPositionBuilderAtEnd(b, b_fetch);
 332
 333                                         /*
 334                                          * If the tupledesc of the to-be-deformed tuple is known,
 335                                          * and JITing of deforming is enabled, build deform
 336                                          * function specific to tupledesc and the exact number of
 337                                          * to-be-extracted attributes.
 338                                          */
 339                                         if (desc && (context->base.flags & PGJIT_DEFORM))
 340                                         {
 341                                                 LLVMValueRef params[1];
 342                                                 LLVMValueRef l_jit_deform;
 343
 344                                                 l_jit_deform =
 345                                                         slot_compile_deform(context, desc,
 346                                                                                                 op->d.fetch.last_var);
 347                                                 params[0] = v_slot;
 348
 349                                                 LLVMBuildCall(b, l_jit_deform,
 350                                                                           params, lengthof(params), "");
 351
 352                                         }
 353                                         else
 354                                         {
 355                                                 LLVMValueRef params[2];
 356
 357                                                 params[0] = v_slot;
 358                                                 params[1] = l_int32_const(op->d.fetch.last_var);
 359
 360                                                 LLVMBuildCall(b,
 361                                                                           llvm_get_decl(mod, FuncSlotGetsomeattrs),
 362                                                                           params, lengthof(params), "");
 363                                         }
 364
 365                                         LLVMBuildBr(b, opblocks[i + 1]);
 366                                         break;
 367                                 }
 368
 369                         case EEOP_INNER_VAR:
 370                         case EEOP_OUTER_VAR:
 371                         case EEOP_SCAN_VAR:
 372                                 {
 373                                         LLVMValueRef value,
 374                                                                 isnull;
 375                                         LLVMValueRef v_attnum;
 376                                         LLVMValueRef v_values;
 377                                         LLVMValueRef v_nulls;
 378
 379                                         if (opcode == EEOP_INNER_VAR)
 380                                         {
 381                                                 v_values = v_innervalues;
 382                                                 v_nulls = v_innernulls;
 383                                         }
 384                                         else if (opcode == EEOP_OUTER_VAR)
 385                                         {
 386                                                 v_values = v_outervalues;
 387                                                 v_nulls = v_outernulls;
 388                                         }
 389                                         else
 390                                         {
 391                                                 v_values = v_scanvalues;
 392                                                 v_nulls = v_scannulls;
 393                                         }
 394
 395                                         v_attnum = l_int32_const(op->d.var.attnum);
 396                                         value = l_load_gep1(b, v_values, v_attnum, "");
 397                                         isnull = l_load_gep1(b, v_nulls, v_attnum, "");
 398                                         LLVMBuildStore(b, value, v_resvaluep);
 399                                         LLVMBuildStore(b, isnull, v_resnullp);
 400
 401                                         LLVMBuildBr(b, opblocks[i + 1]);
 402                                         break;
 403                                 }
 404
 405                         case EEOP_INNER_SYSVAR:
 406                         case EEOP_OUTER_SYSVAR:
 407                         case EEOP_SCAN_SYSVAR:
 408                                 {
 409                                         int                     attnum = op->d.var.attnum;
 410                                         LLVMValueRef v_attnum;
 411                                         LLVMValueRef v_tuple;
 412                                         LLVMValueRef v_tupleDescriptor;
 413                                         LLVMValueRef v_params[4];
 414                                         LLVMValueRef v_syscol;
 415                                         LLVMValueRef v_slot;
 416
 417                                         if (opcode == EEOP_INNER_SYSVAR)
 418                                                 v_slot = v_innerslot;
 419                                         else if (opcode == EEOP_OUTER_SYSVAR)
 420                                                 v_slot = v_outerslot;
 421                                         else
 422                                                 v_slot = v_scanslot;
 423
 424                                         Assert(op->d.var.attnum < 0);
 425
 426                                         v_tuple = l_load_struct_gep(b, v_slot,
 427                                                                                                 FIELDNO_TUPLETABLESLOT_TUPLE,
 428                                                                                                 "v.tuple");
 429
 430                                         /*
 431                                          * Could optimize this a bit for fixed descriptors, but
 432                                          * this shouldn't be that critical a path.
 433                                          */
 434                                         v_tupleDescriptor =
 435                                                 l_load_struct_gep(b, v_slot,
 436                                                                                   FIELDNO_TUPLETABLESLOT_TUPLEDESCRIPTOR,
 437                                                                                   "v.tupledesc");
 438                                         v_attnum = l_int32_const(attnum);
 439
 440                                         v_params[0] = v_tuple;
 441                                         v_params[1] = v_attnum;
 442                                         v_params[2] = v_tupleDescriptor;
 443                                         v_params[3] = v_resnullp;
 444                                         v_syscol = LLVMBuildCall(b,
 445                                                                                          llvm_get_decl(mod, FuncHeapGetsysattr),
 446                                                                                          v_params, lengthof(v_params),
 447                                                                                          "");
 448                                         LLVMBuildStore(b, v_syscol, v_resvaluep);
 449
 450                                         LLVMBuildBr(b, opblocks[i + 1]);
 451                                         break;
 452                                 }
 453
 454                         case EEOP_WHOLEROW:
 455                                 build_EvalXFunc(b, mod, "ExecEvalWholeRowVar",
 456                                                                 v_state, v_econtext, op);
 457                                 LLVMBuildBr(b, opblocks[i + 1]);
 458                                 break;
 459
 460                         case EEOP_ASSIGN_INNER_VAR:
 461                         case EEOP_ASSIGN_OUTER_VAR:
 462                         case EEOP_ASSIGN_SCAN_VAR:
 463                                 {
 464                                         LLVMValueRef v_value,
 465                                                                 v_isnull;
 466                                         LLVMValueRef v_rvaluep,
 467                                                                 v_risnullp;
 468                                         LLVMValueRef v_attnum,
 469                                                                 v_resultnum;
 470                                         LLVMValueRef v_values;
 471                                         LLVMValueRef v_nulls;
 472
 473                                         if (opcode == EEOP_ASSIGN_INNER_VAR)
 474                                         {
 475                                                 v_values = v_innervalues;
 476                                                 v_nulls = v_innernulls;
 477                                         }
 478                                         else if (opcode == EEOP_ASSIGN_OUTER_VAR)
 479                                         {
 480                                                 v_values = v_outervalues;
 481                                                 v_nulls = v_outernulls;
 482                                         }
 483                                         else
 484                                         {
 485                                                 v_values = v_scanvalues;
 486                                                 v_nulls = v_scannulls;
 487                                         }
 488
 489                                         /* load data */
 490                                         v_attnum = l_int32_const(op->d.assign_var.attnum);
 491                                         v_value = l_load_gep1(b, v_values, v_attnum, "");
 492                                         v_isnull = l_load_gep1(b, v_nulls, v_attnum, "");
 493
 494                                         /* compute addresses of targets */
 495                                         v_resultnum = l_int32_const(op->d.assign_var.resultnum);
 496                                         v_rvaluep = LLVMBuildGEP(b, v_resultvalues,
 497                                                                                          &v_resultnum, 1, "");
 498                                         v_risnullp = LLVMBuildGEP(b, v_resultnulls,
 499                                                                                           &v_resultnum, 1, "");
 500
 501                                         /* and store */
 502                                         LLVMBuildStore(b, v_value, v_rvaluep);
 503                                         LLVMBuildStore(b, v_isnull, v_risnullp);
 504
 505                                         LLVMBuildBr(b, opblocks[i + 1]);
 506                                         break;
 507                                 }
 508
 509                         case EEOP_ASSIGN_TMP:
 510                                 {
 511                                         LLVMValueRef v_value,
 512                                                                 v_isnull;
 513                                         LLVMValueRef v_rvaluep,
 514                                                                 v_risnullp;
 515                                         LLVMValueRef v_resultnum;
 516                                         size_t          resultnum = op->d.assign_tmp.resultnum;
 517
 518                                         /* load data */
 519                                         v_value = LLVMBuildLoad(b, v_tmpvaluep, "");
 520                                         v_isnull = LLVMBuildLoad(b, v_tmpisnullp, "");
 521
 522                                         /* compute addresses of targets */
 523                                         v_resultnum = l_int32_const(resultnum);
 524                                         v_rvaluep =
 525                                                 LLVMBuildGEP(b, v_resultvalues, &v_resultnum, 1, "");
 526                                         v_risnullp =
 527                                                 LLVMBuildGEP(b, v_resultnulls, &v_resultnum, 1, "");
 528
 529                                         /* and store */
 530                                         LLVMBuildStore(b, v_value, v_rvaluep);
 531                                         LLVMBuildStore(b, v_isnull, v_risnullp);
 532
 533                                         LLVMBuildBr(b, opblocks[i + 1]);
 534                                         break;
 535                                 }
 536
 537                         case EEOP_ASSIGN_TMP_MAKE_RO:
 538                                 {
 539                                         LLVMBasicBlockRef b_notnull;
 540                                         LLVMValueRef v_params[1];
 541                                         LLVMValueRef v_ret;
 542                                         LLVMValueRef v_value,
 543                                                                 v_isnull;
 544                                         LLVMValueRef v_rvaluep,
 545                                                                 v_risnullp;
 546                                         LLVMValueRef v_resultnum;
 547                                         size_t          resultnum = op->d.assign_tmp.resultnum;
 548
 549                                         b_notnull = l_bb_before_v(opblocks[i + 1],
 550                                                                                           "op.%d.assign_tmp.notnull", i);
 551
 552                                         /* load data */
 553                                         v_value = LLVMBuildLoad(b, v_tmpvaluep, "");
 554                                         v_isnull = LLVMBuildLoad(b, v_tmpisnullp, "");
 555
 556                                         /* compute addresses of targets */
 557                                         v_resultnum = l_int32_const(resultnum);
 558                                         v_rvaluep = LLVMBuildGEP(b, v_resultvalues,
 559                                                                                          &v_resultnum, 1, "");
 560                                         v_risnullp = LLVMBuildGEP(b, v_resultnulls,
 561                                                                                           &v_resultnum, 1, "");
 562
 563                                         /* store nullness */
 564                                         LLVMBuildStore(b, v_isnull, v_risnullp);
 565
 566                                         /* check if value is NULL */
 567                                         LLVMBuildCondBr(b,
 568                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_isnull,
 569                                                                                                   l_sbool_const(0), ""),
 570                                                                         b_notnull, opblocks[i + 1]);
 571
 572                                         /* if value is not null, convert to RO datum */
 573                                         LLVMPositionBuilderAtEnd(b, b_notnull);
 574                                         v_params[0] = v_value;
 575                                         v_ret =
 576                                                 LLVMBuildCall(b,
 577                                                                           llvm_get_decl(mod, FuncMakeExpandedObjectReadOnlyInternal),
 578                                                                           v_params, lengthof(v_params), "");
 579
 580                                         /* store value */
 581                                         LLVMBuildStore(b, v_ret, v_rvaluep);
 582
 583                                         LLVMBuildBr(b, opblocks[i + 1]);
 584                                         break;
 585                                 }
 586
 587                         case EEOP_CONST:
 588                                 {
 589                                         LLVMValueRef v_constvalue,
 590                                                                 v_constnull;
 591
 592                                         v_constvalue = l_sizet_const(op->d.constval.value);
 593                                         v_constnull = l_sbool_const(op->d.constval.isnull);
 594
 595                                         LLVMBuildStore(b, v_constvalue, v_resvaluep);
 596                                         LLVMBuildStore(b, v_constnull, v_resnullp);
 597
 598                                         LLVMBuildBr(b, opblocks[i + 1]);
 599                                         break;
 600                                 }
 601
 602                         case EEOP_FUNCEXPR_STRICT:
 603                                 {
 604                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
 605                                         LLVMBasicBlockRef b_nonull;
 606                                         int                     argno;
 607                                         LLVMValueRef v_fcinfo;
 608                                         LLVMValueRef v_argnullp;
 609                                         LLVMBasicBlockRef *b_checkargnulls;
 610
 611                                         /*
 612                                          * Block for the actual function call, if args are
 613                                          * non-NULL.
 614                                          */
 615                                         b_nonull = l_bb_before_v(opblocks[i + 1],
 616                                                                                          "b.%d.no-null-args", i);
 617
 618                                         /* should make sure they're optimized beforehand */
 619                                         if (op->d.func.nargs == 0)
 620                                                 elog(ERROR, "argumentless strict functions are pointless");
 621
 622                                         v_fcinfo =
 623                                                 l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
 624
 625                                         v_argnullp =
 626                                                 LLVMBuildStructGEP(b,
 627                                                                                    v_fcinfo,
 628                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
 629                                                                                    "v_argnullp");
 630
 631                                         /*
 632                                          * set resnull to true, if the function is actually
 633                                          * called, it'll be reset
 634                                          */
 635                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
 636
 637                                         /* create blocks for checking args, one for each */
 638                                         b_checkargnulls =
 639                                                 palloc(sizeof(LLVMBasicBlockRef *) * op->d.func.nargs);
 640                                         for (argno = 0; argno < op->d.func.nargs; argno++)
 641                                                 b_checkargnulls[argno] =
 642                                                         l_bb_before_v(b_nonull, "b.%d.isnull.%d", i, argno);
 643
 644                                         /* jump to check of first argument */
 645                                         LLVMBuildBr(b, b_checkargnulls[0]);
 646
 647                                         /* check each arg for NULLness */
 648                                         for (argno = 0; argno < op->d.func.nargs; argno++)
 649                                         {
 650                                                 LLVMValueRef v_argisnull;
 651                                                 LLVMBasicBlockRef b_argnotnull;
 652
 653                                                 LLVMPositionBuilderAtEnd(b, b_checkargnulls[argno]);
 654
 655                                                 /* compute block to jump to if argument is not null */
 656                                                 if (argno + 1 == op->d.func.nargs)
 657                                                         b_argnotnull = b_nonull;
 658                                                 else
 659                                                         b_argnotnull = b_checkargnulls[argno + 1];
 660
 661                                                 /* and finally load & check NULLness of arg */
 662                                                 v_argisnull = l_load_struct_gep(b, v_argnullp,
 663                                                                                                                 argno, "");
 664                                                 LLVMBuildCondBr(b,
 665                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
 666                                                                                                           v_argisnull,
 667                                                                                                           l_sbool_const(1),
 668                                                                                                           ""),
 669                                                                                 opblocks[i + 1],
 670                                                                                 b_argnotnull);
 671                                         }
 672
 673                                         LLVMPositionBuilderAtEnd(b, b_nonull);
 674                                 }
 675                                 /* FALLTHROUGH */
 676
 677                         case EEOP_FUNCEXPR:
 678                                 {
 679                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
 680                                         LLVMValueRef v_fcinfo_isnull;
 681                                         LLVMValueRef v_retval;
 682
 683                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
 684                                                                                    &v_fcinfo_isnull);
 685                                         LLVMBuildStore(b, v_retval, v_resvaluep);
 686                                         LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp);
 687
 688                                         LLVMBuildBr(b, opblocks[i + 1]);
 689                                         break;
 690                                 }
 691
 692                         case EEOP_FUNCEXPR_FUSAGE:
 693                                 build_EvalXFunc(b, mod, "ExecEvalFuncExprFusage",
 694                                                                 v_state, v_econtext, op);
 695                                 LLVMBuildBr(b, opblocks[i + 1]);
 696                                 break;
 697
 698
 699                         case EEOP_FUNCEXPR_STRICT_FUSAGE:
 700                                 build_EvalXFunc(b, mod, "ExecEvalFuncExprStrictFusage",
 701                                                                 v_state, v_econtext, op);
 702                                 LLVMBuildBr(b, opblocks[i + 1]);
 703                                 break;
 704
 705                         case EEOP_BOOL_AND_STEP_FIRST:
 706                                 {
 707                                         LLVMValueRef v_boolanynullp;
 708
 709                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 710                                                                                                  l_ptr(TypeStorageBool));
 711                                         LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp);
 712
 713                                 }
 714                                 /* FALLTHROUGH */
 715
 716                                 /*
 717                                  * Treat them the same for now, optimizer can remove
 718                                  * redundancy. Could be worthwhile to optimize during emission
 719                                  * though.
 720                                  */
 721                         case EEOP_BOOL_AND_STEP_LAST:
 722                         case EEOP_BOOL_AND_STEP:
 723                                 {
 724                                         LLVMValueRef v_boolvalue;
 725                                         LLVMValueRef v_boolnull;
 726                                         LLVMValueRef v_boolanynullp,
 727                                                                 v_boolanynull;
 728                                         LLVMBasicBlockRef b_boolisnull;
 729                                         LLVMBasicBlockRef b_boolcheckfalse;
 730                                         LLVMBasicBlockRef b_boolisfalse;
 731                                         LLVMBasicBlockRef b_boolcont;
 732                                         LLVMBasicBlockRef b_boolisanynull;
 733
 734                                         b_boolisnull = l_bb_before_v(opblocks[i + 1],
 735                                                                                                  "b.%d.boolisnull", i);
 736                                         b_boolcheckfalse = l_bb_before_v(opblocks[i + 1],
 737                                                                                                          "b.%d.boolcheckfalse", i);
 738                                         b_boolisfalse = l_bb_before_v(opblocks[i + 1],
 739                                                                                                   "b.%d.boolisfalse", i);
 740                                         b_boolisanynull = l_bb_before_v(opblocks[i + 1],
 741                                                                                                         "b.%d.boolisanynull", i);
 742                                         b_boolcont = l_bb_before_v(opblocks[i + 1],
 743                                                                                            "b.%d.boolcont", i);
 744
 745                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 746                                                                                                  l_ptr(TypeStorageBool));
 747
 748                                         v_boolnull = LLVMBuildLoad(b, v_resnullp, "");
 749                                         v_boolvalue = LLVMBuildLoad(b, v_resvaluep, "");
 750
 751                                         /* set resnull to boolnull */
 752                                         LLVMBuildStore(b, v_boolnull, v_resnullp);
 753                                         /* set revalue to boolvalue */
 754                                         LLVMBuildStore(b, v_boolvalue, v_resvaluep);
 755
 756                                         /* check if current input is NULL */
 757                                         LLVMBuildCondBr(b,
 758                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolnull,
 759                                                                                                   l_sbool_const(1), ""),
 760                                                                         b_boolisnull,
 761                                                                         b_boolcheckfalse);
 762
 763                                         /* build block that sets anynull */
 764                                         LLVMPositionBuilderAtEnd(b, b_boolisnull);
 765                                         /* set boolanynull to true */
 766                                         LLVMBuildStore(b, l_sbool_const(1), v_boolanynullp);
 767                                         /* and jump to next block */
 768                                         LLVMBuildBr(b, b_boolcont);
 769
 770                                         /* build block checking for false */
 771                                         LLVMPositionBuilderAtEnd(b, b_boolcheckfalse);
 772                                         LLVMBuildCondBr(b,
 773                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolvalue,
 774                                                                                                   l_sizet_const(0), ""),
 775                                                                         b_boolisfalse,
 776                                                                         b_boolcont);
 777
 778                                         /*
 779                                          * Build block handling FALSE. Value is false, so short
 780                                          * circuit.
 781                                          */
 782                                         LLVMPositionBuilderAtEnd(b, b_boolisfalse);
 783                                         /* result is already set to FALSE, need not change it */
 784                                         /* and jump to the end of the AND expression */
 785                                         LLVMBuildBr(b, opblocks[op->d.boolexpr.jumpdone]);
 786
 787                                         /* Build block that continues if bool is TRUE. */
 788                                         LLVMPositionBuilderAtEnd(b, b_boolcont);
 789
 790                                         v_boolanynull = LLVMBuildLoad(b, v_boolanynullp, "");
 791
 792                                         /* set value to NULL if any previous values were NULL */
 793                                         LLVMBuildCondBr(b,
 794                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolanynull,
 795                                                                                                   l_sbool_const(0), ""),
 796                                                                         opblocks[i + 1], b_boolisanynull);
 797
 798                                         LLVMPositionBuilderAtEnd(b, b_boolisanynull);
 799                                         /* set resnull to true */
 800                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
 801                                         /* reset resvalue */
 802                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
 803
 804                                         LLVMBuildBr(b, opblocks[i + 1]);
 805                                         break;
 806                                 }
 807                         case EEOP_BOOL_OR_STEP_FIRST:
 808                                 {
 809                                         LLVMValueRef v_boolanynullp;
 810
 811                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 812                                                                                                  l_ptr(TypeStorageBool));
 813                                         LLVMBuildStore(b, l_sbool_const(0), v_boolanynullp);
 814                                 }
 815                                 /* FALLTHROUGH */
 816
 817                                 /*
 818                                  * Treat them the same for now, optimizer can remove
 819                                  * redundancy. Could be worthwhile to optimize during emission
 820                                  * though.
 821                                  */
 822                         case EEOP_BOOL_OR_STEP_LAST:
 823                         case EEOP_BOOL_OR_STEP:
 824                                 {
 825                                         LLVMValueRef v_boolvalue;
 826                                         LLVMValueRef v_boolnull;
 827                                         LLVMValueRef v_boolanynullp,
 828                                                                 v_boolanynull;
 829
 830                                         LLVMBasicBlockRef b_boolisnull;
 831                                         LLVMBasicBlockRef b_boolchecktrue;
 832                                         LLVMBasicBlockRef b_boolistrue;
 833                                         LLVMBasicBlockRef b_boolcont;
 834                                         LLVMBasicBlockRef b_boolisanynull;
 835
 836                                         b_boolisnull = l_bb_before_v(opblocks[i + 1],
 837                                                                                                  "b.%d.boolisnull", i);
 838                                         b_boolchecktrue = l_bb_before_v(opblocks[i + 1],
 839                                                                                                         "b.%d.boolchecktrue", i);
 840                                         b_boolistrue = l_bb_before_v(opblocks[i + 1],
 841                                                                                                  "b.%d.boolistrue", i);
 842                                         b_boolisanynull = l_bb_before_v(opblocks[i + 1],
 843                                                                                                         "b.%d.boolisanynull", i);
 844                                         b_boolcont = l_bb_before_v(opblocks[i + 1],
 845                                                                                            "b.%d.boolcont", i);
 846
 847                                         v_boolanynullp = l_ptr_const(op->d.boolexpr.anynull,
 848                                                                                                  l_ptr(TypeStorageBool));
 849
 850                                         v_boolnull = LLVMBuildLoad(b, v_resnullp, "");
 851                                         v_boolvalue = LLVMBuildLoad(b, v_resvaluep, "");
 852
 853                                         /* set resnull to boolnull */
 854                                         LLVMBuildStore(b, v_boolnull, v_resnullp);
 855                                         /* set revalue to boolvalue */
 856                                         LLVMBuildStore(b, v_boolvalue, v_resvaluep);
 857
 858                                         LLVMBuildCondBr(b,
 859                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolnull,
 860                                                                                                   l_sbool_const(1), ""),
 861                                                                         b_boolisnull,
 862                                                                         b_boolchecktrue);
 863
 864                                         /* build block that sets anynull */
 865                                         LLVMPositionBuilderAtEnd(b, b_boolisnull);
 866                                         /* set boolanynull to true */
 867                                         LLVMBuildStore(b, l_sbool_const(1), v_boolanynullp);
 868                                         /* and jump to next block */
 869                                         LLVMBuildBr(b, b_boolcont);
 870
 871                                         /* build block checking for true */
 872                                         LLVMPositionBuilderAtEnd(b, b_boolchecktrue);
 873                                         LLVMBuildCondBr(b,
 874                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolvalue,
 875                                                                                                   l_sizet_const(1), ""),
 876                                                                         b_boolistrue,
 877                                                                         b_boolcont);
 878
 879                                         /*
 880                                          * Build block handling True. Value is true, so short
 881                                          * circuit.
 882                                          */
 883                                         LLVMPositionBuilderAtEnd(b, b_boolistrue);
 884                                         /* result is already set to TRUE, need not change it */
 885                                         /* and jump to the end of the OR expression */
 886                                         LLVMBuildBr(b, opblocks[op->d.boolexpr.jumpdone]);
 887
 888                                         /* build block that continues if bool is FALSE */
 889                                         LLVMPositionBuilderAtEnd(b, b_boolcont);
 890
 891                                         v_boolanynull = LLVMBuildLoad(b, v_boolanynullp, "");
 892
 893                                         /* set value to NULL if any previous values were NULL */
 894                                         LLVMBuildCondBr(b,
 895                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_boolanynull,
 896                                                                                                   l_sbool_const(0), ""),
 897                                                                         opblocks[i + 1], b_boolisanynull);
 898
 899                                         LLVMPositionBuilderAtEnd(b, b_boolisanynull);
 900                                         /* set resnull to true */
 901                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
 902                                         /* reset resvalue */
 903                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
 904
 905                                         LLVMBuildBr(b, opblocks[i + 1]);
 906                                         break;
 907                                 }
 908
 909                         case EEOP_BOOL_NOT_STEP:
 910                                 {
 911                                         LLVMValueRef v_boolvalue;
 912                                         LLVMValueRef v_boolnull;
 913                                         LLVMValueRef v_negbool;
 914
 915                                         v_boolnull = LLVMBuildLoad(b, v_resnullp, "");
 916                                         v_boolvalue = LLVMBuildLoad(b, v_resvaluep, "");
 917
 918                                         v_negbool = LLVMBuildZExt(b,
 919                                                                                           LLVMBuildICmp(b, LLVMIntEQ,
 920                                                                                                                         v_boolvalue,
 921                                                                                                                         l_sizet_const(0),
 922                                                                                                                         ""),
 923                                                                                           TypeSizeT, "");
 924                                         /* set resnull to boolnull */
 925                                         LLVMBuildStore(b, v_boolnull, v_resnullp);
 926                                         /* set revalue to !boolvalue */
 927                                         LLVMBuildStore(b, v_negbool, v_resvaluep);
 928
 929                                         LLVMBuildBr(b, opblocks[i + 1]);
 930                                         break;
 931                                 }
 932
 933                         case EEOP_QUAL:
 934                                 {
 935                                         LLVMValueRef v_resnull;
 936                                         LLVMValueRef v_resvalue;
 937                                         LLVMValueRef v_nullorfalse;
 938                                         LLVMBasicBlockRef b_qualfail;
 939
 940                                         b_qualfail = l_bb_before_v(opblocks[i + 1],
 941                                                                                            "op.%d.qualfail", i);
 942
 943                                         v_resvalue = LLVMBuildLoad(b, v_resvaluep, "");
 944                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 945
 946                                         v_nullorfalse =
 947                                                 LLVMBuildOr(b,
 948                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
 949                                                                                                   l_sbool_const(1), ""),
 950                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resvalue,
 951                                                                                                   l_sizet_const(0), ""),
 952                                                                         "");
 953
 954                                         LLVMBuildCondBr(b,
 955                                                                         v_nullorfalse,
 956                                                                         b_qualfail,
 957                                                                         opblocks[i + 1]);
 958
 959                                         /* build block handling NULL or false */
 960                                         LLVMPositionBuilderAtEnd(b, b_qualfail);
 961                                         /* set resnull to false */
 962                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
 963                                         /* set resvalue to false */
 964                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
 965                                         /* and jump out */
 966                                         LLVMBuildBr(b, opblocks[op->d.qualexpr.jumpdone]);
 967                                         break;
 968                                 }
 969
 970                         case EEOP_JUMP:
 971                                 {
 972                                         LLVMBuildBr(b, opblocks[op->d.jump.jumpdone]);
 973                                         break;
 974                                 }
 975
 976                         case EEOP_JUMP_IF_NULL:
 977                                 {
 978                                         LLVMValueRef v_resnull;
 979
 980                                         /* Transfer control if current result is null */
 981
 982                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 983
 984                                         LLVMBuildCondBr(b,
 985                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
 986                                                                                                   l_sbool_const(1), ""),
 987                                                                         opblocks[op->d.jump.jumpdone],
 988                                                                         opblocks[i + 1]);
 989                                         break;
 990                                 }
 991
 992                         case EEOP_JUMP_IF_NOT_NULL:
 993                                 {
 994                                         LLVMValueRef v_resnull;
 995
 996                                         /* Transfer control if current result is non-null */
 997
 998                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
 999
1000                                         LLVMBuildCondBr(b,
1001                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1002                                                                                                   l_sbool_const(0), ""),
1003                                                                         opblocks[op->d.jump.jumpdone],
1004                                                                         opblocks[i + 1]);
1005                                         break;
1006                                 }
1007
1008
1009                         case EEOP_JUMP_IF_NOT_TRUE:
1010                                 {
1011                                         LLVMValueRef v_resnull;
1012                                         LLVMValueRef v_resvalue;
1013                                         LLVMValueRef v_nullorfalse;
1014
1015                                         /* Transfer control if current result is null or false */
1016
1017                                         v_resvalue = LLVMBuildLoad(b, v_resvaluep, "");
1018                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1019
1020                                         v_nullorfalse =
1021                                                 LLVMBuildOr(b,
1022                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1023                                                                                                   l_sbool_const(1), ""),
1024                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resvalue,
1025                                                                                                   l_sizet_const(0), ""),
1026                                                                         "");
1027
1028                                         LLVMBuildCondBr(b,
1029                                                                         v_nullorfalse,
1030                                                                         opblocks[op->d.jump.jumpdone],
1031                                                                         opblocks[i + 1]);
1032                                         break;
1033                                 }
1034
1035                         case EEOP_NULLTEST_ISNULL:
1036                                 {
1037                                         LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1038                                         LLVMValueRef v_resvalue;
1039
1040                                         v_resvalue =
1041                                                 LLVMBuildSelect(b,
1042                                                                                 LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1043                                                                                                           l_sbool_const(1), ""),
1044                                                                                 l_sizet_const(1),
1045                                                                                 l_sizet_const(0),
1046                                                                                 "");
1047                                         LLVMBuildStore(b, v_resvalue, v_resvaluep);
1048                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1049
1050                                         LLVMBuildBr(b, opblocks[i + 1]);
1051                                         break;
1052                                 }
1053
1054                         case EEOP_NULLTEST_ISNOTNULL:
1055                                 {
1056                                         LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1057                                         LLVMValueRef v_resvalue;
1058
1059                                         v_resvalue =
1060                                                 LLVMBuildSelect(b,
1061                                                                                 LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1062                                                                                                           l_sbool_const(1), ""),
1063                                                                                 l_sizet_const(0),
1064                                                                                 l_sizet_const(1),
1065                                                                                 "");
1066                                         LLVMBuildStore(b, v_resvalue, v_resvaluep);
1067                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1068
1069                                         LLVMBuildBr(b, opblocks[i + 1]);
1070                                         break;
1071                                 }
1072
1073                         case EEOP_NULLTEST_ROWISNULL:
1074                                 build_EvalXFunc(b, mod, "ExecEvalRowNull",
1075                                                                 v_state, v_econtext, op);
1076                                 LLVMBuildBr(b, opblocks[i + 1]);
1077                                 break;
1078
1079                         case EEOP_NULLTEST_ROWISNOTNULL:
1080                                 build_EvalXFunc(b, mod, "ExecEvalRowNotNull",
1081                                                                 v_state, v_econtext, op);
1082                                 LLVMBuildBr(b, opblocks[i + 1]);
1083                                 break;
1084
1085                         case EEOP_BOOLTEST_IS_TRUE:
1086                         case EEOP_BOOLTEST_IS_NOT_FALSE:
1087                         case EEOP_BOOLTEST_IS_FALSE:
1088                         case EEOP_BOOLTEST_IS_NOT_TRUE:
1089                                 {
1090                                         LLVMBasicBlockRef b_isnull,
1091                                                                 b_notnull;
1092                                         LLVMValueRef v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1093
1094                                         b_isnull = l_bb_before_v(opblocks[i + 1],
1095                                                                                          "op.%d.isnull", i);
1096                                         b_notnull = l_bb_before_v(opblocks[i + 1],
1097                                                                                           "op.%d.isnotnull", i);
1098
1099                                         /* check if value is NULL */
1100                                         LLVMBuildCondBr(b,
1101                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1102                                                                                                   l_sbool_const(1), ""),
1103                                                                         b_isnull, b_notnull);
1104
1105                                         /* if value is NULL, return false */
1106                                         LLVMPositionBuilderAtEnd(b, b_isnull);
1107
1108                                         /* result is not null */
1109                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1110
1111                                         if (opcode == EEOP_BOOLTEST_IS_TRUE ||
1112                                                 opcode == EEOP_BOOLTEST_IS_FALSE)
1113                                         {
1114                                                 LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1115                                         }
1116                                         else
1117                                         {
1118                                                 LLVMBuildStore(b, l_sizet_const(1), v_resvaluep);
1119                                         }
1120
1121                                         LLVMBuildBr(b, opblocks[i + 1]);
1122
1123                                         LLVMPositionBuilderAtEnd(b, b_notnull);
1124
1125                                         if (opcode == EEOP_BOOLTEST_IS_TRUE ||
1126                                                 opcode == EEOP_BOOLTEST_IS_NOT_FALSE)
1127                                         {
1128                                                 /*
1129                                                  * if value is not null NULL, return value (already
1130                                                  * set)
1131                                                  */
1132                                         }
1133                                         else
1134                                         {
1135                                                 LLVMValueRef v_value =
1136                                                 LLVMBuildLoad(b, v_resvaluep, "");
1137
1138                                                 v_value = LLVMBuildZExt(b,
1139                                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
1140                                                                                                                           v_value,
1141                                                                                                                           l_sizet_const(0),
1142                                                                                                                           ""),
1143                                                                                                 TypeSizeT, "");
1144                                                 LLVMBuildStore(b, v_value, v_resvaluep);
1145                                         }
1146                                         LLVMBuildBr(b, opblocks[i + 1]);
1147                                         break;
1148                                 }
1149
1150                         case EEOP_PARAM_EXEC:
1151                                 build_EvalXFunc(b, mod, "ExecEvalParamExec",
1152                                                                 v_state, v_econtext, op);
1153                                 LLVMBuildBr(b, opblocks[i + 1]);
1154                                 break;
1155
1156                         case EEOP_PARAM_EXTERN:
1157                                 build_EvalXFunc(b, mod, "ExecEvalParamExtern",
1158                                                                 v_state, v_econtext, op);
1159                                 LLVMBuildBr(b, opblocks[i + 1]);
1160                                 break;
1161
1162                         case EEOP_PARAM_CALLBACK:
1163                                 {
1164                                         LLVMTypeRef param_types[3];
1165                                         LLVMValueRef v_params[3];
1166                                         LLVMTypeRef v_functype;
1167                                         LLVMValueRef v_func;
1168
1169                                         param_types[0] = l_ptr(StructExprState);
1170                                         param_types[1] = l_ptr(TypeSizeT);
1171                                         param_types[2] = l_ptr(StructExprContext);
1172
1173                                         v_functype = LLVMFunctionType(LLVMVoidType(),
1174                                                                                                   param_types,
1175                                                                                                   lengthof(param_types),
1176                                                                                                   false);
1177                                         v_func = l_ptr_const(op->d.cparam.paramfunc,
1178                                                                                  l_ptr(v_functype));
1179
1180                                         v_params[0] = v_state;
1181                                         v_params[1] = l_ptr_const(op, l_ptr(TypeSizeT));
1182                                         v_params[2] = v_econtext;
1183                                         LLVMBuildCall(b,
1184                                                                   v_func,
1185                                                                   v_params, lengthof(v_params), "");
1186
1187                                         LLVMBuildBr(b, opblocks[i + 1]);
1188                                         break;
1189                                 }
1190
1191                         case EEOP_ARRAYREF_OLD:
1192                                 build_EvalXFunc(b, mod, "ExecEvalArrayRefOld",
1193                                                                 v_state, v_econtext, op);
1194                                 LLVMBuildBr(b, opblocks[i + 1]);
1195                                 break;
1196
1197                         case EEOP_ARRAYREF_ASSIGN:
1198                                 build_EvalXFunc(b, mod, "ExecEvalArrayRefAssign",
1199                                                                 v_state, v_econtext, op);
1200                                 LLVMBuildBr(b, opblocks[i + 1]);
1201                                 break;
1202
1203                         case EEOP_ARRAYREF_FETCH:
1204                                 build_EvalXFunc(b, mod, "ExecEvalArrayRefFetch",
1205                                                                 v_state, v_econtext, op);
1206                                 LLVMBuildBr(b, opblocks[i + 1]);
1207                                 break;
1208
1209                         case EEOP_CASE_TESTVAL:
1210                                 {
1211                                         LLVMBasicBlockRef b_avail,
1212                                                                 b_notavail;
1213                                         LLVMValueRef v_casevaluep,
1214                                                                 v_casevalue;
1215                                         LLVMValueRef v_casenullp,
1216                                                                 v_casenull;
1217                                         LLVMValueRef v_casevaluenull;
1218
1219                                         b_avail = l_bb_before_v(opblocks[i + 1],
1220                                                                                         "op.%d.avail", i);
1221                                         b_notavail = l_bb_before_v(opblocks[i + 1],
1222                                                                                            "op.%d.notavail", i);
1223
1224                                         v_casevaluep = l_ptr_const(op->d.casetest.value,
1225                                                                                            l_ptr(TypeSizeT));
1226                                         v_casenullp = l_ptr_const(op->d.casetest.isnull,
1227                                                                                           l_ptr(TypeStorageBool));
1228
1229                                         v_casevaluenull =
1230                                                 LLVMBuildICmp(b, LLVMIntEQ,
1231                                                                           LLVMBuildPtrToInt(b, v_casevaluep,
1232                                                                                                                 TypeSizeT, ""),
1233                                                                           l_sizet_const(0), "");
1234                                         LLVMBuildCondBr(b, v_casevaluenull, b_notavail, b_avail);
1235
1236                                         /* if casetest != NULL */
1237                                         LLVMPositionBuilderAtEnd(b, b_avail);
1238                                         v_casevalue = LLVMBuildLoad(b, v_casevaluep, "");
1239                                         v_casenull = LLVMBuildLoad(b, v_casenullp, "");
1240                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1241                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1242                                         LLVMBuildBr(b, opblocks[i + 1]);
1243
1244                                         /* if casetest == NULL */
1245                                         LLVMPositionBuilderAtEnd(b, b_notavail);
1246                                         v_casevalue =
1247                                                 l_load_struct_gep(b, v_econtext,
1248                                                                                   FIELDNO_EXPRCONTEXT_CASEDATUM, "");
1249                                         v_casenull =
1250                                                 l_load_struct_gep(b, v_econtext,
1251                                                                                   FIELDNO_EXPRCONTEXT_CASENULL, "");
1252                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1253                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1254
1255                                         LLVMBuildBr(b, opblocks[i + 1]);
1256                                         break;
1257                                 }
1258
1259                         case EEOP_MAKE_READONLY:
1260                                 {
1261                                         LLVMBasicBlockRef b_notnull;
1262                                         LLVMValueRef v_params[1];
1263                                         LLVMValueRef v_ret;
1264                                         LLVMValueRef v_nullp;
1265                                         LLVMValueRef v_valuep;
1266                                         LLVMValueRef v_null;
1267                                         LLVMValueRef v_value;
1268
1269                                         b_notnull = l_bb_before_v(opblocks[i + 1],
1270                                                                                           "op.%d.readonly.notnull", i);
1271
1272                                         v_nullp = l_ptr_const(op->d.make_readonly.isnull,
1273                                                                                   l_ptr(TypeStorageBool));
1274
1275                                         v_null = LLVMBuildLoad(b, v_nullp, "");
1276
1277                                         /* store null isnull value in result */
1278                                         LLVMBuildStore(b, v_null, v_resnullp);
1279
1280                                         /* check if value is NULL */
1281                                         LLVMBuildCondBr(b,
1282                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_null,
1283                                                                                                   l_sbool_const(1), ""),
1284                                                                         opblocks[i + 1], b_notnull);
1285
1286                                         /* if value is not null, convert to RO datum */
1287                                         LLVMPositionBuilderAtEnd(b, b_notnull);
1288
1289                                         v_valuep = l_ptr_const(op->d.make_readonly.value,
1290                                                                                    l_ptr(TypeSizeT));
1291
1292                                         v_value = LLVMBuildLoad(b, v_valuep, "");
1293
1294                                         v_params[0] = v_value;
1295                                         v_ret =
1296                                                 LLVMBuildCall(b,
1297                                                                           llvm_get_decl(mod, FuncMakeExpandedObjectReadOnlyInternal),
1298                                                                           v_params, lengthof(v_params), "");
1299                                         LLVMBuildStore(b, v_ret, v_resvaluep);
1300
1301                                         LLVMBuildBr(b, opblocks[i + 1]);
1302                                         break;
1303                                 }
1304
1305                         case EEOP_IOCOERCE:
1306                                 {
1307                                         FunctionCallInfo fcinfo_out,
1308                                                                 fcinfo_in;
1309                                         LLVMValueRef v_fcinfo_out,
1310                                                                 v_fcinfo_in;
1311                                         LLVMValueRef v_fn_addr_out,
1312                                                                 v_fn_addr_in;
1313                                         LLVMValueRef v_fcinfo_in_isnullp;
1314                                         LLVMValueRef v_in_argp,
1315                                                                 v_out_argp;
1316                                         LLVMValueRef v_in_argnullp,
1317                                                                 v_out_argnullp;
1318                                         LLVMValueRef v_retval;
1319                                         LLVMValueRef v_resvalue;
1320                                         LLVMValueRef v_resnull;
1321
1322                                         LLVMValueRef v_output_skip;
1323                                         LLVMValueRef v_output;
1324
1325                                         LLVMBasicBlockRef b_skipoutput;
1326                                         LLVMBasicBlockRef b_calloutput;
1327                                         LLVMBasicBlockRef b_input;
1328                                         LLVMBasicBlockRef b_inputcall;
1329
1330                                         fcinfo_out = op->d.iocoerce.fcinfo_data_out;
1331                                         fcinfo_in = op->d.iocoerce.fcinfo_data_in;
1332
1333                                         b_skipoutput = l_bb_before_v(opblocks[i + 1],
1334                                                                                                  "op.%d.skipoutputnull", i);
1335                                         b_calloutput = l_bb_before_v(opblocks[i + 1],
1336                                                                                                  "op.%d.calloutput", i);
1337                                         b_input = l_bb_before_v(opblocks[i + 1],
1338                                                                                         "op.%d.input", i);
1339                                         b_inputcall = l_bb_before_v(opblocks[i + 1],
1340                                                                                                 "op.%d.inputcall", i);
1341
1342                                         v_fcinfo_out = l_ptr_const(fcinfo_out, l_ptr(StructFunctionCallInfoData));
1343                                         v_fcinfo_in = l_ptr_const(fcinfo_in, l_ptr(StructFunctionCallInfoData));
1344                                         v_fn_addr_out = l_ptr_const(fcinfo_out->flinfo->fn_addr, TypePGFunction);
1345                                         v_fn_addr_in = l_ptr_const(fcinfo_in->flinfo->fn_addr, TypePGFunction);
1346
1347                                         v_fcinfo_in_isnullp =
1348                                                 LLVMBuildStructGEP(b, v_fcinfo_in,
1349                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ISNULL,
1350                                                                                    "v_fcinfo_in_isnull");
1351                                         v_out_argnullp =
1352                                                 LLVMBuildStructGEP(b, v_fcinfo_out,
1353                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
1354                                                                                    "v_fcinfo_out_argnullp");
1355                                         v_in_argnullp =
1356                                                 LLVMBuildStructGEP(b, v_fcinfo_in,
1357                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
1358                                                                                    "v_fcinfo_in_argnullp");
1359                                         v_out_argp =
1360                                                 LLVMBuildStructGEP(b, v_fcinfo_out,
1361                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARG,
1362                                                                                    "v_fcinfo_out_argp");
1363                                         v_in_argp =
1364                                                 LLVMBuildStructGEP(b, v_fcinfo_in,
1365                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARG,
1366                                                                                    "v_fcinfo_in_argp");
1367
1368                                         /* output functions are not called on nulls */
1369                                         v_resnull = LLVMBuildLoad(b, v_resnullp, "");
1370                                         LLVMBuildCondBr(b,
1371                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_resnull,
1372                                                                                                   l_sbool_const(1), ""),
1373                                                                         b_skipoutput,
1374                                                                         b_calloutput);
1375
1376                                         LLVMPositionBuilderAtEnd(b, b_skipoutput);
1377                                         v_output_skip = l_sizet_const(0);
1378                                         LLVMBuildBr(b, b_input);
1379
1380                                         LLVMPositionBuilderAtEnd(b, b_calloutput);
1381                                         v_resvalue = LLVMBuildLoad(b, v_resvaluep, "");
1382
1383                                         /* set arg[0] */
1384                                         LLVMBuildStore(b,
1385                                                                    v_resvalue,
1386                                                                    LLVMBuildStructGEP(b, v_out_argp, 0, ""));
1387                                         LLVMBuildStore(b,
1388                                                                    l_sbool_const(0),
1389                                                                    LLVMBuildStructGEP(b, v_out_argnullp,
1390                                                                                                           0, ""));
1391                                         /* and call output function (can never return NULL) */
1392                                         v_output = LLVMBuildCall(b, v_fn_addr_out, &v_fcinfo_out,
1393                                                                                          1, "funccall_coerce_out");
1394                                         LLVMBuildBr(b, b_input);
1395
1396                                         /* build block handling input function call */
1397                                         LLVMPositionBuilderAtEnd(b, b_input);
1398
1399                                         /* phi between resnull and output function call branches */
1400                                         {
1401                                                 LLVMValueRef incoming_values[2];
1402                                                 LLVMBasicBlockRef incoming_blocks[2];
1403
1404                                                 incoming_values[0] = v_output_skip;
1405                                                 incoming_blocks[0] = b_skipoutput;
1406
1407                                                 incoming_values[1] = v_output;
1408                                                 incoming_blocks[1] = b_calloutput;
1409
1410                                                 v_output = LLVMBuildPhi(b, TypeSizeT, "output");
1411                                                 LLVMAddIncoming(v_output,
1412                                                                                 incoming_values, incoming_blocks,
1413                                                                                 lengthof(incoming_blocks));
1414                                         }
1415
1416                                         /*
1417                                          * If input function is strict, skip if input string is
1418                                          * NULL.
1419                                          */
1420                                         if (op->d.iocoerce.finfo_in->fn_strict)
1421                                         {
1422                                                 LLVMBuildCondBr(b,
1423                                                                                 LLVMBuildICmp(b, LLVMIntEQ, v_output,
1424                                                                                                           l_sizet_const(0), ""),
1425                                                                                 opblocks[i + 1],
1426                                                                                 b_inputcall);
1427                                         }
1428                                         else
1429                                         {
1430                                                 LLVMBuildBr(b, b_inputcall);
1431                                         }
1432
1433                                         LLVMPositionBuilderAtEnd(b, b_inputcall);
1434                                         /* set arguments */
1435                                         /* arg0: output */
1436                                         LLVMBuildStore(b, v_output,
1437                                                                    LLVMBuildStructGEP(b, v_in_argp, 0, ""));
1438                                         LLVMBuildStore(b, v_resnull,
1439                                                                    LLVMBuildStructGEP(b, v_in_argnullp, 0, ""));
1440
1441                                         /* arg1: ioparam: preset in execExpr.c */
1442                                         /* arg2: typmod: preset in execExpr.c  */
1443
1444                                         /* reset fcinfo_in->isnull */
1445                                         LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_in_isnullp);
1446                                         /* and call function */
1447                                         v_retval = LLVMBuildCall(b, v_fn_addr_in, &v_fcinfo_in, 1,
1448                                                                                          "funccall_iocoerce_in");
1449
1450                                         LLVMBuildStore(b, v_retval, v_resvaluep);
1451
1452                                         LLVMBuildBr(b, opblocks[i + 1]);
1453                                         break;
1454                                 }
1455
1456                         case EEOP_DISTINCT:
1457                         case EEOP_NOT_DISTINCT:
1458                                 {
1459                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
1460
1461                                         LLVMValueRef v_fcinfo;
1462                                         LLVMValueRef v_fcinfo_isnull;
1463
1464                                         LLVMValueRef v_argnullp;
1465                                         LLVMValueRef v_argnull0,
1466                                                                 v_argisnull0;
1467                                         LLVMValueRef v_argnull1,
1468                                                                 v_argisnull1;
1469
1470                                         LLVMValueRef v_anyargisnull;
1471                                         LLVMValueRef v_bothargisnull;
1472
1473                                         LLVMValueRef v_result;
1474
1475                                         LLVMBasicBlockRef b_noargnull;
1476                                         LLVMBasicBlockRef b_checkbothargnull;
1477                                         LLVMBasicBlockRef b_bothargnull;
1478                                         LLVMBasicBlockRef b_anyargnull;
1479
1480                                         b_noargnull = l_bb_before_v(opblocks[i + 1], "op.%d.noargnull", i);
1481                                         b_checkbothargnull = l_bb_before_v(opblocks[i + 1], "op.%d.checkbothargnull", i);
1482                                         b_bothargnull = l_bb_before_v(opblocks[i + 1], "op.%d.bothargnull", i);
1483                                         b_anyargnull = l_bb_before_v(opblocks[i + 1], "op.%d.anyargnull", i);
1484
1485                                         v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
1486
1487                                         v_argnullp =
1488                                                 LLVMBuildStructGEP(b,
1489                                                                                    v_fcinfo,
1490                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
1491                                                                                    "v_argnullp");
1492
1493                                         /* load argnull[0|1] for both arguments */
1494                                         v_argnull0 = l_load_struct_gep(b, v_argnullp, 0, "");
1495                                         v_argisnull0 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull0,
1496                                                                                                  l_sbool_const(1), "");
1497
1498                                         v_argnull1 = l_load_struct_gep(b, v_argnullp, 1, "");
1499                                         v_argisnull1 = LLVMBuildICmp(b, LLVMIntEQ, v_argnull1,
1500                                                                                                  l_sbool_const(1), "");
1501
1502                                         v_anyargisnull = LLVMBuildOr(b, v_argisnull0, v_argisnull1, "");
1503                                         v_bothargisnull = LLVMBuildAnd(b, v_argisnull0, v_argisnull1, "");
1504
1505                                         /*
1506                                          * Check function arguments for NULLness: If either is
1507                                          * NULL, we check if both args are NULL. Otherwise call
1508                                          * comparator.
1509                                          */
1510                                         LLVMBuildCondBr(b, v_anyargisnull, b_checkbothargnull,
1511                                                                         b_noargnull);
1512
1513                                         /*
1514                                          * build block checking if any arg is null
1515                                          */
1516                                         LLVMPositionBuilderAtEnd(b, b_checkbothargnull);
1517                                         LLVMBuildCondBr(b, v_bothargisnull, b_bothargnull,
1518                                                                         b_anyargnull);
1519
1520
1521                                         /* Both NULL? Then is not distinct... */
1522                                         LLVMPositionBuilderAtEnd(b, b_bothargnull);
1523                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1524                                         if (opcode == EEOP_NOT_DISTINCT)
1525                                                 LLVMBuildStore(b, l_sizet_const(1), v_resvaluep);
1526                                         else
1527                                                 LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1528
1529                                         LLVMBuildBr(b, opblocks[i + 1]);
1530
1531                                         /* Only one is NULL? Then is distinct... */
1532                                         LLVMPositionBuilderAtEnd(b, b_anyargnull);
1533                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1534                                         if (opcode == EEOP_NOT_DISTINCT)
1535                                                 LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1536                                         else
1537                                                 LLVMBuildStore(b, l_sizet_const(1), v_resvaluep);
1538                                         LLVMBuildBr(b, opblocks[i + 1]);
1539
1540                                         /* neither argument is null: compare */
1541                                         LLVMPositionBuilderAtEnd(b, b_noargnull);
1542
1543                                         v_result = BuildV1Call(context, b, mod, fcinfo,
1544                                                                                    &v_fcinfo_isnull);
1545
1546                                         if (opcode == EEOP_DISTINCT)
1547                                         {
1548                                                 /* Must invert result of "=" */
1549                                                 v_result =
1550                                                         LLVMBuildZExt(b,
1551                                                                                   LLVMBuildICmp(b, LLVMIntEQ,
1552                                                                                                                 v_result,
1553                                                                                                                 l_sizet_const(0), ""),
1554                                                                                   TypeSizeT, "");
1555                                         }
1556
1557                                         LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp);
1558                                         LLVMBuildStore(b, v_result, v_resvaluep);
1559
1560                                         LLVMBuildBr(b, opblocks[i + 1]);
1561                                         break;
1562                                 }
1563
1564                         case EEOP_NULLIF:
1565                                 {
1566                                         FunctionCallInfo fcinfo = op->d.func.fcinfo_data;
1567
1568                                         LLVMValueRef v_fcinfo;
1569                                         LLVMValueRef v_fcinfo_isnull;
1570                                         LLVMValueRef v_argnullp;
1571                                         LLVMValueRef v_argnull0;
1572                                         LLVMValueRef v_argnull1;
1573                                         LLVMValueRef v_anyargisnull;
1574                                         LLVMValueRef v_argp;
1575                                         LLVMValueRef v_arg0;
1576                                         LLVMBasicBlockRef b_hasnull;
1577                                         LLVMBasicBlockRef b_nonull;
1578                                         LLVMBasicBlockRef b_argsequal;
1579                                         LLVMValueRef v_retval;
1580                                         LLVMValueRef v_argsequal;
1581
1582                                         b_hasnull = l_bb_before_v(opblocks[i + 1],
1583                                                                                           "b.%d.null-args", i);
1584                                         b_nonull = l_bb_before_v(opblocks[i + 1],
1585                                                                                          "b.%d.no-null-args", i);
1586                                         b_argsequal = l_bb_before_v(opblocks[i + 1],
1587                                                                                                 "b.%d.argsequal", i);
1588
1589                                         v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
1590
1591                                         v_argnullp =
1592                                                 LLVMBuildStructGEP(b,
1593                                                                                    v_fcinfo,
1594                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
1595                                                                                    "v_argnullp");
1596
1597                                         v_argp =
1598                                                 LLVMBuildStructGEP(b,
1599                                                                                    v_fcinfo,
1600                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARG,
1601                                                                                    "v_argp");
1602
1603                                         /* if either argument is NULL they can't be equal */
1604                                         v_argnull0 = l_load_struct_gep(b, v_argnullp, 0, "");
1605                                         v_argnull1 = l_load_struct_gep(b, v_argnullp, 1, "");
1606
1607                                         v_anyargisnull =
1608                                                 LLVMBuildOr(b,
1609                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_argnull0,
1610                                                                                                   l_sbool_const(1), ""),
1611                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_argnull1,
1612                                                                                                   l_sbool_const(1), ""),
1613                                                                         "");
1614
1615                                         LLVMBuildCondBr(b, v_anyargisnull, b_hasnull, b_nonull);
1616
1617                                         /* one (or both) of the arguments are null, return arg[0] */
1618                                         LLVMPositionBuilderAtEnd(b, b_hasnull);
1619                                         v_arg0 = l_load_struct_gep(b, v_argp, 0, "");
1620                                         LLVMBuildStore(b, v_argnull0, v_resnullp);
1621                                         LLVMBuildStore(b, v_arg0, v_resvaluep);
1622                                         LLVMBuildBr(b, opblocks[i + 1]);
1623
1624                                         /* build block to invoke function and check result */
1625                                         LLVMPositionBuilderAtEnd(b, b_nonull);
1626
1627                                         v_retval = BuildV1Call(context, b, mod, fcinfo, &v_fcinfo_isnull);
1628
1629                                         /*
1630                                          * If result not null, and arguments are equal return null
1631                                          * (same result as if there'd been NULLs, hence reuse
1632                                          * b_hasnull).
1633                                          */
1634                                         v_argsequal = LLVMBuildAnd(b,
1635                                                                                            LLVMBuildICmp(b, LLVMIntEQ,
1636                                                                                                                          v_fcinfo_isnull,
1637                                                                                                                          l_sbool_const(0),
1638                                                                                                                          ""),
1639                                                                                            LLVMBuildICmp(b, LLVMIntEQ,
1640                                                                                                                          v_retval,
1641                                                                                                                          l_sizet_const(1),
1642                                                                                                                          ""),
1643                                                                                            "");
1644                                         LLVMBuildCondBr(b, v_argsequal, b_argsequal, b_hasnull);
1645
1646                                         /* build block setting result to NULL, if args are equal */
1647                                         LLVMPositionBuilderAtEnd(b, b_argsequal);
1648                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
1649                                         LLVMBuildStore(b, l_sizet_const(0), v_resvaluep);
1650                                         LLVMBuildStore(b, v_retval, v_resvaluep);
1651
1652                                         LLVMBuildBr(b, opblocks[i + 1]);
1653                                         break;
1654                                 }
1655
1656                         case EEOP_SQLVALUEFUNCTION:
1657                                 build_EvalXFunc(b, mod, "ExecEvalSQLValueFunction",
1658                                                                 v_state, v_econtext, op);
1659                                 LLVMBuildBr(b, opblocks[i + 1]);
1660                                 break;
1661
1662                         case EEOP_CURRENTOFEXPR:
1663                                 build_EvalXFunc(b, mod, "ExecEvalCurrentOfExpr",
1664                                                                 v_state, v_econtext, op);
1665                                 LLVMBuildBr(b, opblocks[i + 1]);
1666                                 break;
1667
1668                         case EEOP_NEXTVALUEEXPR:
1669                                 build_EvalXFunc(b, mod, "ExecEvalNextValueExpr",
1670                                                                 v_state, v_econtext, op);
1671                                 LLVMBuildBr(b, opblocks[i + 1]);
1672                                 break;
1673
1674                         case EEOP_ARRAYEXPR:
1675                                 build_EvalXFunc(b, mod, "ExecEvalArrayExpr",
1676                                                                 v_state, v_econtext, op);
1677                                 LLVMBuildBr(b, opblocks[i + 1]);
1678                                 break;
1679
1680                         case EEOP_ARRAYCOERCE:
1681                                 build_EvalXFunc(b, mod, "ExecEvalArrayCoerce",
1682                                                                 v_state, v_econtext, op);
1683                                 LLVMBuildBr(b, opblocks[i + 1]);
1684                                 break;
1685
1686                         case EEOP_ROW:
1687                                 build_EvalXFunc(b, mod, "ExecEvalRow",
1688                                                                 v_state, v_econtext, op);
1689                                 LLVMBuildBr(b, opblocks[i + 1]);
1690                                 break;
1691
1692                         case EEOP_ROWCOMPARE_STEP:
1693                                 {
1694                                         FunctionCallInfo fcinfo = op->d.rowcompare_step.fcinfo_data;
1695                                         LLVMValueRef v_fcinfo_isnull;
1696                                         LLVMBasicBlockRef b_null;
1697                                         LLVMBasicBlockRef b_compare;
1698                                         LLVMBasicBlockRef b_compare_result;
1699
1700                                         LLVMValueRef v_retval;
1701
1702                                         b_null = l_bb_before_v(opblocks[i + 1],
1703                                                                                    "op.%d.row-null", i);
1704                                         b_compare = l_bb_before_v(opblocks[i + 1],
1705                                                                                           "op.%d.row-compare", i);
1706                                         b_compare_result =
1707                                                 l_bb_before_v(opblocks[i + 1],
1708                                                                           "op.%d.row-compare-result",
1709                                                                           i);
1710
1711                                         /*
1712                                          * If function is strict, and either arg is null, we're
1713                                          * done.
1714                                          */
1715                                         if (op->d.rowcompare_step.finfo->fn_strict)
1716                                         {
1717                                                 LLVMValueRef v_fcinfo;
1718                                                 LLVMValueRef v_argnullp;
1719                                                 LLVMValueRef v_argnull0;
1720                                                 LLVMValueRef v_argnull1;
1721                                                 LLVMValueRef v_anyargisnull;
1722
1723                                                 v_fcinfo = l_ptr_const(fcinfo,
1724                                                                                            l_ptr(StructFunctionCallInfoData));
1725
1726                                                 v_argnullp =
1727                                                         LLVMBuildStructGEP(b, v_fcinfo,
1728                                                                                            FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
1729                                                                                            "v_argnullp");
1730
1731                                                 v_argnull0 = l_load_struct_gep(b, v_argnullp, 0, "");
1732                                                 v_argnull1 = l_load_struct_gep(b, v_argnullp, 1, "");
1733
1734                                                 v_anyargisnull =
1735                                                         LLVMBuildOr(b,
1736                                                                                 LLVMBuildICmp(b,
1737                                                                                                           LLVMIntEQ,
1738                                                                                                           v_argnull0,
1739                                                                                                           l_sbool_const(1),
1740                                                                                                           ""),
1741                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
1742                                                                                                           v_argnull1,
1743                                                                                                           l_sbool_const(1), ""),
1744                                                                                 "");
1745
1746                                                 LLVMBuildCondBr(b, v_anyargisnull, b_null, b_compare);
1747                                         }
1748                                         else
1749                                         {
1750                                                 LLVMBuildBr(b, b_compare);
1751                                         }
1752
1753                                         /* build block invoking comparison function */
1754                                         LLVMPositionBuilderAtEnd(b, b_compare);
1755
1756                                         /* call function */
1757                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
1758                                                                                    &v_fcinfo_isnull);
1759                                         LLVMBuildStore(b, v_retval, v_resvaluep);
1760
1761                                         /* if result of function is NULL, force NULL result */
1762                                         LLVMBuildCondBr(b,
1763                                                                         LLVMBuildICmp(b,
1764                                                                                                   LLVMIntEQ,
1765                                                                                                   v_fcinfo_isnull,
1766                                                                                                   l_sbool_const(0),
1767                                                                                                   ""),
1768                                                                         b_compare_result,
1769                                                                         b_null);
1770
1771                                         /* build block analyzing the !NULL comparator result */
1772                                         LLVMPositionBuilderAtEnd(b, b_compare_result);
1773
1774                                         /* if results equal, compare next, otherwise done */
1775                                         LLVMBuildCondBr(b,
1776                                                                         LLVMBuildICmp(b,
1777                                                                                                   LLVMIntEQ,
1778                                                                                                   v_retval,
1779                                                                                                   l_sizet_const(0), ""),
1780                                                                         opblocks[i + 1],
1781                                                                         opblocks[op->d.rowcompare_step.jumpdone]);
1782
1783                                         /*
1784                                          * Build block handling NULL input or NULL comparator
1785                                          * result.
1786                                          */
1787                                         LLVMPositionBuilderAtEnd(b, b_null);
1788                                         LLVMBuildStore(b, l_sbool_const(1), v_resnullp);
1789                                         LLVMBuildBr(b, opblocks[op->d.rowcompare_step.jumpnull]);
1790
1791                                         break;
1792                                 }
1793
1794                         case EEOP_ROWCOMPARE_FINAL:
1795                                 {
1796                                         RowCompareType rctype = op->d.rowcompare_final.rctype;
1797
1798                                         LLVMValueRef v_cmpresult;
1799                                         LLVMValueRef v_result;
1800                                         LLVMIntPredicate predicate;
1801
1802                                         /*
1803                                          * Btree comparators return 32 bit results, need to be
1804                                          * careful about sign (used as a 64 bit value it's
1805                                          * otherwise wrong).
1806                                          */
1807                                         v_cmpresult =
1808                                                 LLVMBuildTrunc(b,
1809                                                                            LLVMBuildLoad(b, v_resvaluep, ""),
1810                                                                            LLVMInt32Type(), "");
1811
1812                                         switch (rctype)
1813                                         {
1814                                                 case ROWCOMPARE_LT:
1815                                                         predicate = LLVMIntSLT;
1816                                                         break;
1817                                                 case ROWCOMPARE_LE:
1818                                                         predicate = LLVMIntSLE;
1819                                                         break;
1820                                                 case ROWCOMPARE_GT:
1821                                                         predicate = LLVMIntSGT;
1822                                                         break;
1823                                                 case ROWCOMPARE_GE:
1824                                                         predicate = LLVMIntSGE;
1825                                                         break;
1826                                                 default:
1827                                                         /* EQ and NE cases aren't allowed here */
1828                                                         Assert(false);
1829                                                         predicate = 0;  /* prevent compiler warning */
1830                                                         break;
1831                                         }
1832
1833                                         v_result = LLVMBuildICmp(b,
1834                                                                                          predicate,
1835                                                                                          v_cmpresult,
1836                                                                                          l_int32_const(0),
1837                                                                                          "");
1838                                         v_result = LLVMBuildZExt(b, v_result, TypeSizeT, "");
1839
1840                                         LLVMBuildStore(b, l_sbool_const(0), v_resnullp);
1841                                         LLVMBuildStore(b, v_result, v_resvaluep);
1842
1843                                         LLVMBuildBr(b, opblocks[i + 1]);
1844                                         break;
1845                                 }
1846
1847                         case EEOP_MINMAX:
1848                                 build_EvalXFunc(b, mod, "ExecEvalMinMax",
1849                                                                 v_state, v_econtext, op);
1850                                 LLVMBuildBr(b, opblocks[i + 1]);
1851                                 break;
1852
1853                         case EEOP_FIELDSELECT:
1854                                 build_EvalXFunc(b, mod, "ExecEvalFieldSelect",
1855                                                                 v_state, v_econtext, op);
1856                                 LLVMBuildBr(b, opblocks[i + 1]);
1857                                 break;
1858
1859                         case EEOP_FIELDSTORE_DEFORM:
1860                                 build_EvalXFunc(b, mod, "ExecEvalFieldStoreDeForm",
1861                                                                 v_state, v_econtext, op);
1862                                 LLVMBuildBr(b, opblocks[i + 1]);
1863                                 break;
1864
1865                         case EEOP_FIELDSTORE_FORM:
1866                                 build_EvalXFunc(b, mod, "ExecEvalFieldStoreForm",
1867                                                                 v_state, v_econtext, op);
1868                                 LLVMBuildBr(b, opblocks[i + 1]);
1869                                 break;
1870
1871                         case EEOP_ARRAYREF_SUBSCRIPT:
1872                                 {
1873                                         LLVMValueRef v_fn;
1874                                         int                     jumpdone = op->d.arrayref_subscript.jumpdone;
1875                                         LLVMValueRef v_params[2];
1876                                         LLVMValueRef v_ret;
1877
1878                                         v_fn = llvm_get_decl(mod, FuncExecEvalArrayRefSubscript);
1879
1880                                         v_params[0] = v_state;
1881                                         v_params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
1882                                         v_ret = LLVMBuildCall(b, v_fn,
1883                                                                                   v_params, lengthof(v_params), "");
1884                                         v_ret = LLVMBuildZExt(b, v_ret, TypeStorageBool, "");
1885
1886                                         LLVMBuildCondBr(b,
1887                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_ret,
1888                                                                                                   l_sbool_const(1), ""),
1889                                                                         opblocks[i + 1],
1890                                                                         opblocks[jumpdone]);
1891                                         break;
1892                                 }
1893
1894                         case EEOP_DOMAIN_TESTVAL:
1895                                 {
1896                                         LLVMBasicBlockRef b_avail,
1897                                                                 b_notavail;
1898                                         LLVMValueRef v_casevaluep,
1899                                                                 v_casevalue;
1900                                         LLVMValueRef v_casenullp,
1901                                                                 v_casenull;
1902                                         LLVMValueRef v_casevaluenull;
1903
1904                                         b_avail = l_bb_before_v(opblocks[i + 1],
1905                                                                                         "op.%d.avail", i);
1906                                         b_notavail = l_bb_before_v(opblocks[i + 1],
1907                                                                                            "op.%d.notavail", i);
1908
1909                                         v_casevaluep = l_ptr_const(op->d.casetest.value,
1910                                                                                            l_ptr(TypeSizeT));
1911                                         v_casenullp = l_ptr_const(op->d.casetest.isnull,
1912                                                                                           l_ptr(TypeStorageBool));
1913
1914                                         v_casevaluenull =
1915                                                 LLVMBuildICmp(b, LLVMIntEQ,
1916                                                                           LLVMBuildPtrToInt(b, v_casevaluep,
1917                                                                                                                 TypeSizeT, ""),
1918                                                                           l_sizet_const(0), "");
1919                                         LLVMBuildCondBr(b,
1920                                                                         v_casevaluenull,
1921                                                                         b_notavail, b_avail);
1922
1923                                         /* if casetest != NULL */
1924                                         LLVMPositionBuilderAtEnd(b, b_avail);
1925                                         v_casevalue = LLVMBuildLoad(b, v_casevaluep, "");
1926                                         v_casenull = LLVMBuildLoad(b, v_casenullp, "");
1927                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1928                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1929                                         LLVMBuildBr(b, opblocks[i + 1]);
1930
1931                                         /* if casetest == NULL */
1932                                         LLVMPositionBuilderAtEnd(b, b_notavail);
1933                                         v_casevalue =
1934                                                 l_load_struct_gep(b, v_econtext,
1935                                                                                   FIELDNO_EXPRCONTEXT_DOMAINDATUM,
1936                                                                                   "");
1937                                         v_casenull =
1938                                                 l_load_struct_gep(b, v_econtext,
1939                                                                                   FIELDNO_EXPRCONTEXT_DOMAINNULL,
1940                                                                                   "");
1941                                         LLVMBuildStore(b, v_casevalue, v_resvaluep);
1942                                         LLVMBuildStore(b, v_casenull, v_resnullp);
1943
1944                                         LLVMBuildBr(b, opblocks[i + 1]);
1945                                         break;
1946                                 }
1947
1948                         case EEOP_DOMAIN_NOTNULL:
1949                                 build_EvalXFunc(b, mod, "ExecEvalConstraintNotNull",
1950                                                                 v_state, v_econtext, op);
1951                                 LLVMBuildBr(b, opblocks[i + 1]);
1952                                 break;
1953
1954                         case EEOP_DOMAIN_CHECK:
1955                                 build_EvalXFunc(b, mod, "ExecEvalConstraintCheck",
1956                                                                 v_state, v_econtext, op);
1957                                 LLVMBuildBr(b, opblocks[i + 1]);
1958                                 break;
1959
1960                         case EEOP_CONVERT_ROWTYPE:
1961                                 build_EvalXFunc(b, mod, "ExecEvalConvertRowtype",
1962                                                                 v_state, v_econtext, op);
1963                                 LLVMBuildBr(b, opblocks[i + 1]);
1964                                 break;
1965
1966                         case EEOP_SCALARARRAYOP:
1967                                 build_EvalXFunc(b, mod, "ExecEvalScalarArrayOp",
1968                                                                 v_state, v_econtext, op);
1969                                 LLVMBuildBr(b, opblocks[i + 1]);
1970                                 break;
1971
1972                         case EEOP_XMLEXPR:
1973                                 build_EvalXFunc(b, mod, "ExecEvalXmlExpr",
1974                                                                 v_state, v_econtext, op);
1975                                 LLVMBuildBr(b, opblocks[i + 1]);
1976                                 break;
1977
1978                         case EEOP_AGGREF:
1979                                 {
1980                                         AggrefExprState *aggref = op->d.aggref.astate;
1981                                         LLVMValueRef v_aggnop;
1982                                         LLVMValueRef v_aggno;
1983                                         LLVMValueRef value,
1984                                                                 isnull;
1985
1986                                         /*
1987                                          * At this point aggref->aggno is not yet set (it's set up
1988                                          * in ExecInitAgg() after initializing the expression). So
1989                                          * load it from memory each time round.
1990                                          */
1991                                         v_aggnop = l_ptr_const(&aggref->aggno,
1992                                                                                    l_ptr(LLVMInt32Type()));
1993                                         v_aggno = LLVMBuildLoad(b, v_aggnop, "v_aggno");
1994
1995                                         /* load agg value / null */
1996                                         value = l_load_gep1(b, v_aggvalues, v_aggno, "aggvalue");
1997                                         isnull = l_load_gep1(b, v_aggnulls, v_aggno, "aggnull");
1998
1999                                         /* and store result */
2000                                         LLVMBuildStore(b, value, v_resvaluep);
2001                                         LLVMBuildStore(b, isnull, v_resnullp);
2002
2003                                         LLVMBuildBr(b, opblocks[i + 1]);
2004                                         break;
2005                                 }
2006
2007                         case EEOP_GROUPING_FUNC:
2008                                 build_EvalXFunc(b, mod, "ExecEvalGroupingFunc",
2009                                                                 v_state, v_econtext, op);
2010                                 LLVMBuildBr(b, opblocks[i + 1]);
2011                                 break;
2012
2013                         case EEOP_WINDOW_FUNC:
2014                                 {
2015                                         WindowFuncExprState *wfunc = op->d.window_func.wfstate;
2016                                         LLVMValueRef v_wfuncnop;
2017                                         LLVMValueRef v_wfuncno;
2018                                         LLVMValueRef value,
2019                                                                 isnull;
2020
2021                                         /*
2022                                          * At this point aggref->wfuncno is not yet set (it's set
2023                                          * up in ExecInitWindowAgg() after initializing the
2024                                          * expression). So load it from memory each time round.
2025                                          */
2026                                         v_wfuncnop = l_ptr_const(&wfunc->wfuncno,
2027                                                                                          l_ptr(LLVMInt32Type()));
2028                                         v_wfuncno = LLVMBuildLoad(b, v_wfuncnop, "v_wfuncno");
2029
2030                                         /* load window func value / null */
2031                                         value = l_load_gep1(b, v_aggvalues, v_wfuncno,
2032                                                                                 "windowvalue");
2033                                         isnull = l_load_gep1(b, v_aggnulls, v_wfuncno,
2034                                                                                  "windownull");
2035
2036                                         LLVMBuildStore(b, value, v_resvaluep);
2037                                         LLVMBuildStore(b, isnull, v_resnullp);
2038
2039                                         LLVMBuildBr(b, opblocks[i + 1]);
2040                                         break;
2041                                 }
2042
2043                         case EEOP_SUBPLAN:
2044                                 build_EvalXFunc(b, mod, "ExecEvalSubPlan",
2045                                                                 v_state, v_econtext, op);
2046                                 LLVMBuildBr(b, opblocks[i + 1]);
2047                                 break;
2048
2049                         case EEOP_ALTERNATIVE_SUBPLAN:
2050                                 build_EvalXFunc(b, mod, "ExecEvalAlternativeSubPlan",
2051                                                                 v_state, v_econtext, op);
2052                                 LLVMBuildBr(b, opblocks[i + 1]);
2053                                 break;
2054
2055                         case EEOP_AGG_STRICT_DESERIALIZE:
2056                                 {
2057                                         FunctionCallInfo fcinfo = op->d.agg_deserialize.fcinfo_data;
2058                                         LLVMValueRef v_fcinfo;
2059                                         LLVMValueRef v_argnullp;
2060                                         LLVMValueRef v_argnull0;
2061                                         LLVMBasicBlockRef b_deserialize;
2062
2063                                         b_deserialize = l_bb_before_v(opblocks[i + 1],
2064                                                                                                   "op.%d.deserialize", i);
2065
2066                                         v_fcinfo = l_ptr_const(fcinfo,
2067                                                                                    l_ptr(StructFunctionCallInfoData));
2068
2069                                         v_argnullp =
2070                                                 LLVMBuildStructGEP(b,
2071                                                                                    v_fcinfo,
2072                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
2073                                                                                    "v_argnullp");
2074                                         v_argnull0 =
2075                                                 l_load_struct_gep(b, v_argnullp, 0, "v_argnull0");
2076
2077                                         LLVMBuildCondBr(b,
2078                                                                         LLVMBuildICmp(b,
2079                                                                                                   LLVMIntEQ,
2080                                                                                                   v_argnull0,
2081                                                                                                   l_sbool_const(1),
2082                                                                                                   ""),
2083                                                                         opblocks[op->d.agg_deserialize.jumpnull],
2084                                                                         b_deserialize);
2085                                         LLVMPositionBuilderAtEnd(b, b_deserialize);
2086                                 }
2087                                 /* FALLTHROUGH */
2088
2089                         case EEOP_AGG_DESERIALIZE:
2090                                 {
2091                                         AggState   *aggstate;
2092                                         FunctionCallInfo fcinfo;
2093
2094                                         LLVMValueRef v_retval;
2095                                         LLVMValueRef v_fcinfo_isnull;
2096                                         LLVMValueRef v_tmpcontext;
2097                                         LLVMValueRef v_oldcontext;
2098
2099                                         aggstate = op->d.agg_deserialize.aggstate;
2100                                         fcinfo = op->d.agg_deserialize.fcinfo_data;
2101
2102                                         v_tmpcontext =
2103                                                 l_ptr_const(aggstate->tmpcontext->ecxt_per_tuple_memory,
2104                                                                         l_ptr(StructMemoryContextData));
2105                                         v_oldcontext = l_mcxt_switch(mod, b, v_tmpcontext);
2106                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
2107                                                                                    &v_fcinfo_isnull);
2108                                         l_mcxt_switch(mod, b, v_oldcontext);
2109
2110                                         LLVMBuildStore(b, v_retval, v_resvaluep);
2111                                         LLVMBuildStore(b, v_fcinfo_isnull, v_resnullp);
2112
2113                                         LLVMBuildBr(b, opblocks[i + 1]);
2114                                         break;
2115                                 }
2116
2117                         case EEOP_AGG_STRICT_INPUT_CHECK:
2118                                 {
2119                                         int                     nargs = op->d.agg_strict_input_check.nargs;
2120                                         bool       *nulls = op->d.agg_strict_input_check.nulls;
2121                                         int                     jumpnull;
2122                                         int                     argno;
2123
2124                                         LLVMValueRef v_nullp;
2125                                         LLVMBasicBlockRef *b_checknulls;
2126
2127                                         jumpnull = op->d.agg_strict_input_check.jumpnull;
2128                                         v_nullp = l_ptr_const(nulls, l_ptr(TypeStorageBool));
2129
2130                                         /* create blocks for checking args */
2131                                         b_checknulls = palloc(sizeof(LLVMBasicBlockRef *) * nargs);
2132                                         for (argno = 0; argno < nargs; argno++)
2133                                         {
2134                                                 b_checknulls[argno] =
2135                                                         l_bb_before_v(opblocks[i + 1],
2136                                                                                   "op.%d.check-null.%d",
2137                                                                                   i, argno);
2138                                         }
2139
2140                                         LLVMBuildBr(b, b_checknulls[0]);
2141
2142                                         /* strict function, check for NULL args */
2143                                         for (argno = 0; argno < nargs; argno++)
2144                                         {
2145                                                 LLVMValueRef v_argno = l_int32_const(argno);
2146                                                 LLVMValueRef v_argisnull;
2147                                                 LLVMBasicBlockRef b_argnotnull;
2148
2149                                                 LLVMPositionBuilderAtEnd(b, b_checknulls[argno]);
2150
2151                                                 if (argno + 1 == nargs)
2152                                                         b_argnotnull = opblocks[i + 1];
2153                                                 else
2154                                                         b_argnotnull = b_checknulls[argno + 1];
2155
2156                                                 v_argisnull = l_load_gep1(b, v_nullp, v_argno, "");
2157
2158                                                 LLVMBuildCondBr(b,
2159                                                                                 LLVMBuildICmp(b,
2160                                                                                                           LLVMIntEQ,
2161                                                                                                           v_argisnull,
2162                                                                                                           l_sbool_const(1), ""),
2163                                                                                 opblocks[jumpnull],
2164                                                                                 b_argnotnull);
2165                                         }
2166
2167                                         break;
2168                                 }
2169
2170                         case EEOP_AGG_INIT_TRANS:
2171                                 {
2172                                         AggState   *aggstate;
2173                                         AggStatePerTrans pertrans;
2174
2175                                         LLVMValueRef v_aggstatep;
2176                                         LLVMValueRef v_pertransp;
2177
2178                                         LLVMValueRef v_allpergroupsp;
2179
2180                                         LLVMValueRef v_pergroupp;
2181
2182                                         LLVMValueRef v_setoff,
2183                                                                 v_transno;
2184
2185                                         LLVMValueRef v_notransvalue;
2186
2187                                         LLVMBasicBlockRef b_init;
2188
2189                                         aggstate = op->d.agg_init_trans.aggstate;
2190                                         pertrans = op->d.agg_init_trans.pertrans;
2191
2192                                         v_aggstatep = l_ptr_const(aggstate,
2193                                                                                           l_ptr(StructAggState));
2194                                         v_pertransp = l_ptr_const(pertrans,
2195                                                                                           l_ptr(StructAggStatePerTransData));
2196
2197                                         /*
2198                                          * pergroup = &aggstate->all_pergroups
2199                                          * [op->d.agg_init_trans_check.setoff]
2200                                          * [op->d.agg_init_trans_check.transno];
2201                                          */
2202                                         v_allpergroupsp =
2203                                                 l_load_struct_gep(b, v_aggstatep,
2204                                                                                   FIELDNO_AGGSTATE_ALL_PERGROUPS,
2205                                                                                   "aggstate.all_pergroups");
2206                                         v_setoff = l_int32_const(op->d.agg_init_trans.setoff);
2207                                         v_transno = l_int32_const(op->d.agg_init_trans.transno);
2208                                         v_pergroupp =
2209                                                 LLVMBuildGEP(b,
2210                                                                          l_load_gep1(b, v_allpergroupsp, v_setoff, ""),
2211                                                                          &v_transno, 1, "");
2212
2213                                         v_notransvalue =
2214                                                 l_load_struct_gep(b, v_pergroupp,
2215                                                                                   FIELDNO_AGGSTATEPERGROUPDATA_NOTRANSVALUE,
2216                                                                                   "notransvalue");
2217
2218                                         b_init = l_bb_before_v(opblocks[i + 1],
2219                                                                                    "op.%d.inittrans", i);
2220
2221                                         LLVMBuildCondBr(b,
2222                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_notransvalue,
2223                                                                                                   l_sbool_const(1), ""),
2224                                                                         b_init,
2225                                                                         opblocks[i + 1]);
2226
2227                                         LLVMPositionBuilderAtEnd(b, b_init);
2228
2229                                         {
2230                                                 LLVMValueRef params[3];
2231
2232                                                 params[0] = v_aggstatep;
2233                                                 params[1] = v_pertransp;
2234                                                 params[2] = v_pergroupp;
2235
2236                                                 LLVMBuildCall(b,
2237                                                                           llvm_get_decl(mod, FuncExecAggInitGroup),
2238                                                                           params, lengthof(params),
2239                                                                           "");
2240                                         }
2241                                         LLVMBuildBr(b, opblocks[op->d.agg_init_trans.jumpnull]);
2242
2243                                         break;
2244                                 }
2245
2246                         case EEOP_AGG_STRICT_TRANS_CHECK:
2247                                 {
2248                                         AggState   *aggstate;
2249                                         LLVMValueRef v_setoff,
2250                                                                 v_transno;
2251
2252                                         LLVMValueRef v_aggstatep;
2253                                         LLVMValueRef v_allpergroupsp;
2254
2255                                         LLVMValueRef v_transnull;
2256                                         LLVMValueRef v_pergroupp;
2257
2258                                         int                     jumpnull = op->d.agg_strict_trans_check.jumpnull;
2259
2260                                         aggstate = op->d.agg_strict_trans_check.aggstate;
2261                                         v_aggstatep = l_ptr_const(aggstate, l_ptr(StructAggState));
2262
2263                                         /*
2264                                          * pergroup = &aggstate->all_pergroups
2265                                          * [op->d.agg_strict_trans_check.setoff]
2266                                          * [op->d.agg_init_trans_check.transno];
2267                                          */
2268                                         v_allpergroupsp =
2269                                                 l_load_struct_gep(b, v_aggstatep,
2270                                                                                   FIELDNO_AGGSTATE_ALL_PERGROUPS,
2271                                                                                   "aggstate.all_pergroups");
2272                                         v_setoff =
2273                                                 l_int32_const(op->d.agg_strict_trans_check.setoff);
2274                                         v_transno =
2275                                                 l_int32_const(op->d.agg_strict_trans_check.transno);
2276                                         v_pergroupp =
2277                                                 LLVMBuildGEP(b,
2278                                                                          l_load_gep1(b, v_allpergroupsp, v_setoff, ""),
2279                                                                          &v_transno, 1, "");
2280
2281                                         v_transnull =
2282                                                 l_load_struct_gep(b, v_pergroupp,
2283                                                                                   FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL,
2284                                                                                   "transnull");
2285
2286                                         LLVMBuildCondBr(b,
2287                                                                         LLVMBuildICmp(b, LLVMIntEQ, v_transnull,
2288                                                                                                   l_sbool_const(1), ""),
2289                                                                         opblocks[jumpnull],
2290                                                                         opblocks[i + 1]);
2291
2292                                         break;
2293                                 }
2294
2295                         case EEOP_AGG_PLAIN_TRANS_BYVAL:
2296                         case EEOP_AGG_PLAIN_TRANS:
2297                                 {
2298                                         AggState   *aggstate;
2299                                         AggStatePerTrans pertrans;
2300                                         FunctionCallInfo fcinfo;
2301
2302                                         LLVMValueRef v_aggstatep;
2303                                         LLVMValueRef v_fcinfo;
2304                                         LLVMValueRef v_fcinfo_isnull;
2305                                         LLVMValueRef v_argp,
2306                                                                 v_argnullp;
2307
2308                                         LLVMValueRef v_transvaluep;
2309                                         LLVMValueRef v_transnullp;
2310
2311                                         LLVMValueRef v_setoff;
2312                                         LLVMValueRef v_transno;
2313
2314                                         LLVMValueRef v_aggcontext;
2315
2316                                         LLVMValueRef v_allpergroupsp;
2317                                         LLVMValueRef v_current_setp;
2318                                         LLVMValueRef v_current_pertransp;
2319                                         LLVMValueRef v_curaggcontext;
2320
2321                                         LLVMValueRef v_pertransp;
2322
2323                                         LLVMValueRef v_pergroupp;
2324
2325                                         LLVMValueRef v_retval;
2326
2327                                         LLVMValueRef v_tmpcontext;
2328                                         LLVMValueRef v_oldcontext;
2329
2330                                         aggstate = op->d.agg_trans.aggstate;
2331                                         pertrans = op->d.agg_trans.pertrans;
2332
2333                                         fcinfo = &pertrans->transfn_fcinfo;
2334
2335                                         v_aggstatep = l_ptr_const(aggstate,
2336                                                                                           l_ptr(StructAggState));
2337                                         v_pertransp = l_ptr_const(pertrans,
2338                                                                                           l_ptr(StructAggStatePerTransData));
2339
2340                                         /*
2341                                          * pergroup = &aggstate->all_pergroups
2342                                          * [op->d.agg_strict_trans_check.setoff]
2343                                          * [op->d.agg_init_trans_check.transno];
2344                                          */
2345                                         v_allpergroupsp =
2346                                                 l_load_struct_gep(b, v_aggstatep,
2347                                                                                   FIELDNO_AGGSTATE_ALL_PERGROUPS,
2348                                                                                   "aggstate.all_pergroups");
2349                                         v_setoff = l_int32_const(op->d.agg_trans.setoff);
2350                                         v_transno = l_int32_const(op->d.agg_trans.transno);
2351                                         v_pergroupp =
2352                                                 LLVMBuildGEP(b,
2353                                                                          l_load_gep1(b, v_allpergroupsp, v_setoff, ""),
2354                                                                          &v_transno, 1, "");
2355
2356                                         v_fcinfo = l_ptr_const(fcinfo,
2357                                                                                    l_ptr(StructFunctionCallInfoData));
2358
2359                                         v_argnullp =
2360                                                 LLVMBuildStructGEP(b,
2361                                                                                    v_fcinfo,
2362                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARGNULL,
2363                                                                                    "v_argnullp");
2364                                         v_argp =
2365                                                 LLVMBuildStructGEP(b,
2366                                                                                    v_fcinfo,
2367                                                                                    FIELDNO_FUNCTIONCALLINFODATA_ARG,
2368                                                                                    "v_argp");
2369
2370                                         v_aggcontext = l_ptr_const(op->d.agg_trans.aggcontext,
2371                                                                                            l_ptr(StructExprContext));
2372
2373                                         v_current_setp =
2374                                                 LLVMBuildStructGEP(b,
2375                                                                                    v_aggstatep,
2376                                                                                    FIELDNO_AGGSTATE_CURRENT_SET,
2377                                                                                    "aggstate.current_set");
2378                                         v_curaggcontext =
2379                                                 LLVMBuildStructGEP(b,
2380                                                                                    v_aggstatep,
2381                                                                                    FIELDNO_AGGSTATE_CURAGGCONTEXT,
2382                                                                                    "aggstate.curaggcontext");
2383                                         v_current_pertransp =
2384                                                 LLVMBuildStructGEP(b,
2385                                                                                    v_aggstatep,
2386                                                                                    FIELDNO_AGGSTATE_CURPERTRANS,
2387                                                                                    "aggstate.curpertrans");
2388
2389                                         /* set aggstate globals */
2390                                         LLVMBuildStore(b, v_aggcontext, v_curaggcontext);
2391                                         LLVMBuildStore(b, l_int32_const(op->d.agg_trans.setno),
2392                                                                    v_current_setp);
2393                                         LLVMBuildStore(b, v_pertransp, v_current_pertransp);
2394
2395                                         /* invoke transition function in per-tuple context */
2396                                         v_tmpcontext =
2397                                                 l_ptr_const(aggstate->tmpcontext->ecxt_per_tuple_memory,
2398                                                                         l_ptr(StructMemoryContextData));
2399                                         v_oldcontext = l_mcxt_switch(mod, b, v_tmpcontext);
2400
2401                                         /* store transvalue in fcinfo->arg/argnull[0] */
2402                                         v_transvaluep =
2403                                                 LLVMBuildStructGEP(b, v_pergroupp,
2404                                                                                    FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUE,
2405                                                                                    "transvalue");
2406                                         v_transnullp =
2407                                                 LLVMBuildStructGEP(b, v_pergroupp,
2408                                                                                    FIELDNO_AGGSTATEPERGROUPDATA_TRANSVALUEISNULL,
2409                                                                                    "transnullp");
2410                                         LLVMBuildStore(b,
2411                                                                    LLVMBuildLoad(b, v_transvaluep,
2412                                                                                                  "transvalue"),
2413                                                                    LLVMBuildStructGEP(b, v_argp, 0, ""));
2414                                         LLVMBuildStore(b,
2415                                                                    LLVMBuildLoad(b, v_transnullp, "transnull"),
2416                                                                    LLVMBuildStructGEP(b, v_argnullp, 0, ""));
2417
2418                                         /* and invoke transition function */
2419                                         v_retval = BuildV1Call(context, b, mod, fcinfo,
2420                                                                                    &v_fcinfo_isnull);
2421
2422                                         /*
2423                                          * For pass-by-ref datatype, must copy the new value into
2424                                          * aggcontext and free the prior transValue.  But if
2425                                          * transfn returned a pointer to its first input, we don't
2426                                          * need to do anything.  Also, if transfn returned a
2427                                          * pointer to a R/W expanded object that is already a
2428                                          * child of the aggcontext, assume we can adopt that value
2429                                          * without copying it.
2430                                          */
2431                                         if (opcode == EEOP_AGG_PLAIN_TRANS)
2432                                         {
2433                                                 LLVMBasicBlockRef b_call;
2434                                                 LLVMBasicBlockRef b_nocall;
2435                                                 LLVMValueRef v_fn;
2436                                                 LLVMValueRef v_transvalue;
2437                                                 LLVMValueRef v_transnull;
2438                                                 LLVMValueRef v_newval;
2439                                                 LLVMValueRef params[6];
2440
2441                                                 b_call = l_bb_before_v(opblocks[i + 1],
2442                                                                                            "op.%d.transcall", i);
2443                                                 b_nocall = l_bb_before_v(opblocks[i + 1],
2444                                                                                                  "op.%d.transnocall", i);
2445
2446                                                 v_transvalue = LLVMBuildLoad(b, v_transvaluep, "");
2447                                                 v_transnull = LLVMBuildLoad(b, v_transnullp, "");
2448
2449                                                 /*
2450                                                  * DatumGetPointer(newVal) !=
2451                                                  * DatumGetPointer(pergroup->transValue))
2452                                                  */
2453                                                 LLVMBuildCondBr(b,
2454                                                                                 LLVMBuildICmp(b, LLVMIntEQ,
2455                                                                                                           v_transvalue,
2456                                                                                                           v_retval, ""),
2457                                                                                 b_nocall, b_call);
2458
2459                                                 /* returned datum not passed datum, reparent */
2460                                                 LLVMPositionBuilderAtEnd(b, b_call);
2461
2462                                                 params[0] = v_aggstatep;
2463                                                 params[1] = v_pertransp;
2464                                                 params[2] = v_retval;
2465                                                 params[3] = LLVMBuildTrunc(b, v_fcinfo_isnull,
2466                                                                                                    TypeParamBool, "");
2467                                                 params[4] = v_transvalue;
2468                                                 params[5] = LLVMBuildTrunc(b, v_transnull,
2469                                                                                                    TypeParamBool, "");
2470
2471                                                 v_fn = llvm_get_decl(mod, FuncExecAggTransReparent);
2472                                                 v_newval =
2473                                                         LLVMBuildCall(b, v_fn,
2474                                                                                   params, lengthof(params),
2475                                                                                   "");
2476
2477                                                 /* store trans value */
2478                                                 LLVMBuildStore(b, v_newval, v_transvaluep);
2479                                                 LLVMBuildStore(b, v_fcinfo_isnull, v_transnullp);
2480                                                 LLVMBuildBr(b, opblocks[i + 1]);
2481
2482                                                 /* returned datum passed datum, no need to reparent */
2483                                                 LLVMPositionBuilderAtEnd(b, b_nocall);
2484                                         }
2485
2486                                         /* store trans value */
2487                                         LLVMBuildStore(b, v_retval, v_transvaluep);
2488                                         LLVMBuildStore(b, v_fcinfo_isnull, v_transnullp);
2489
2490                                         l_mcxt_switch(mod, b, v_oldcontext);
2491
2492                                         LLVMBuildBr(b, opblocks[i + 1]);
2493                                         break;
2494                                 }
2495
2496                         case EEOP_AGG_ORDERED_TRANS_DATUM:
2497                                 build_EvalXFunc(b, mod, "ExecEvalAggOrderedTransDatum",
2498                                                                 v_state, v_econtext, op);
2499                                 LLVMBuildBr(b, opblocks[i + 1]);
2500                                 break;
2501
2502                         case EEOP_AGG_ORDERED_TRANS_TUPLE:
2503                                 build_EvalXFunc(b, mod, "ExecEvalAggOrderedTransTuple",
2504                                                                 v_state, v_econtext, op);
2505                                 LLVMBuildBr(b, opblocks[i + 1]);
2506                                 break;
2507
2508                         case EEOP_LAST:
2509                                 Assert(false);
2510                                 break;
2511                 }
2512         }
2513
2514         LLVMDisposeBuilder(b);
2515
2516         /*
2517          * Don't immediately emit function, instead do so the first time the
2518          * expression is actually evaluated. That allows to emit a lot of
2519          * functions together, avoiding a lot of repeated llvm and memory
2520          * remapping overhead.
2521          */
2522         {
2523
2524                 CompiledExprState *cstate = palloc0(sizeof(CompiledExprState));
2525
2526                 cstate->context = context;
2527                 cstate->funcname = funcname;
2528
2529                 state->evalfunc = ExecRunCompiledExpr;
2530                 state->evalfunc_private = cstate;
2531         }
2532
2533         llvm_leave_fatal_on_oom();
2534
2535         INSTR_TIME_SET_CURRENT(endtime);
2536         INSTR_TIME_ACCUM_DIFF(context->base.generation_counter,
2537                                                   endtime, starttime);
2538
2539         return true;
2540 }
2541
2542 /*
2543  * Run compiled expression.
2544  *
2545  * This will only be called the first time a JITed expression is called. We
2546  * first make sure the expression is still up2date, and then get a pointer to
2547  * the emitted function. The latter can be the first thing that triggers
2548  * optimizing and emitting all the generated functions.
2549  */
2550 static Datum
2551 ExecRunCompiledExpr(ExprState *state, ExprContext *econtext, bool *isNull)
2552 {
2553         CompiledExprState *cstate = state->evalfunc_private;
2554         ExprStateEvalFunc func;
2555
2556         CheckExprStillValid(state, econtext);
2557
2558         llvm_enter_fatal_on_oom();
2559         func = (ExprStateEvalFunc) llvm_get_function(cstate->context,
2560                                                                                                  cstate->funcname);
2561         llvm_leave_fatal_on_oom();
2562         Assert(func);
2563
2564         /* remove indirection via this function for future calls */
2565         state->evalfunc = func;
2566
2567         return func(state, econtext, isNull);
2568 }
2569
2570 static LLVMValueRef
2571 BuildV1Call(LLVMJitContext *context, LLVMBuilderRef b,
2572                         LLVMModuleRef mod, FunctionCallInfo fcinfo,
2573                         LLVMValueRef *v_fcinfo_isnull)
2574 {
2575         LLVMValueRef v_fn;
2576         LLVMValueRef v_fcinfo_isnullp;
2577         LLVMValueRef v_retval;
2578         LLVMValueRef v_fcinfo;
2579
2580         v_fn = llvm_function_reference(context, b, mod, fcinfo);
2581
2582         v_fcinfo = l_ptr_const(fcinfo, l_ptr(StructFunctionCallInfoData));
2583         v_fcinfo_isnullp = LLVMBuildStructGEP(b, v_fcinfo,
2584                                                                                   FIELDNO_FUNCTIONCALLINFODATA_ISNULL,
2585                                                                                   "v_fcinfo_isnull");
2586         LLVMBuildStore(b, l_sbool_const(0), v_fcinfo_isnullp);
2587
2588         v_retval = LLVMBuildCall(b, v_fn, &v_fcinfo, 1, "funccall");
2589
2590         if (v_fcinfo_isnull)
2591                 *v_fcinfo_isnull = LLVMBuildLoad(b, v_fcinfo_isnullp, "");
2592
2593         /*
2594          * Add lifetime-end annotation, signalling that writes to memory don't
2595          * have to be retained (important for inlining potential).
2596          */
2597         {
2598                 LLVMValueRef v_lifetime = create_LifetimeEnd(mod);
2599                 LLVMValueRef params[2];
2600
2601                 params[0] = l_int64_const(sizeof(fcinfo->arg));
2602                 params[1] = l_ptr_const(fcinfo->arg, l_ptr(LLVMInt8Type()));
2603                 LLVMBuildCall(b, v_lifetime, params, lengthof(params), "");
2604
2605                 params[0] = l_int64_const(sizeof(fcinfo->argnull));
2606                 params[1] = l_ptr_const(fcinfo->argnull, l_ptr(LLVMInt8Type()));
2607                 LLVMBuildCall(b, v_lifetime, params, lengthof(params), "");
2608
2609                 params[0] = l_int64_const(sizeof(fcinfo->isnull));
2610                 params[1] = l_ptr_const(&fcinfo->isnull, l_ptr(LLVMInt8Type()));
2611                 LLVMBuildCall(b, v_lifetime, params, lengthof(params), "");
2612         }
2613
2614         return v_retval;
2615 }
2616
2617 /*
2618  * Implement an expression step by calling the function funcname.
2619  */
2620 static void
2621 build_EvalXFunc(LLVMBuilderRef b, LLVMModuleRef mod, const char *funcname,
2622                                 LLVMValueRef v_state, LLVMValueRef v_econtext,
2623                                 ExprEvalStep *op)
2624 {
2625         LLVMTypeRef sig;
2626         LLVMValueRef v_fn;
2627         LLVMTypeRef param_types[3];
2628         LLVMValueRef params[3];
2629
2630         v_fn = LLVMGetNamedFunction(mod, funcname);
2631         if (!v_fn)
2632         {
2633                 param_types[0] = l_ptr(StructExprState);
2634                 param_types[1] = l_ptr(StructExprEvalStep);
2635                 param_types[2] = l_ptr(StructExprContext);
2636
2637                 sig = LLVMFunctionType(LLVMVoidType(),
2638                                                            param_types, lengthof(param_types),
2639                                                            false);
2640                 v_fn = LLVMAddFunction(mod, funcname, sig);
2641         }
2642
2643         params[0] = v_state;
2644         params[1] = l_ptr_const(op, l_ptr(StructExprEvalStep));
2645         params[2] = v_econtext;
2646
2647         LLVMBuildCall(b,
2648                                   v_fn,
2649                                   params, lengthof(params), "");
2650 }
2651
2652 static LLVMValueRef
2653 create_LifetimeEnd(LLVMModuleRef mod)
2654 {
2655         LLVMTypeRef sig;
2656         LLVMValueRef fn;
2657         LLVMTypeRef param_types[2];
2658
2659         /* LLVM 5+ has a variadic pointer argument */
2660 #if LLVM_VERSION_MAJOR < 5
2661         const char *nm = "llvm.lifetime.end";
2662 #else
2663         const char *nm = "llvm.lifetime.end.p0i8";
2664 #endif
2665
2666         fn = LLVMGetNamedFunction(mod, nm);
2667         if (fn)
2668                 return fn;
2669
2670         param_types[0] = LLVMInt64Type();
2671         param_types[1] = l_ptr(LLVMInt8Type());
2672
2673         sig = LLVMFunctionType(LLVMVoidType(),
2674                                                    param_types, lengthof(param_types),
2675                                                    false);
2676         fn = LLVMAddFunction(mod, nm, sig);
2677
2678         LLVMSetFunctionCallConv(fn, LLVMCCallConv);
2679
2680         Assert(LLVMGetIntrinsicID(fn));
2681
2682         return fn;
2683 }