1 /*-------------------------------------------------------------------------
4 * functions for gathering statistics from tsvector columns
6 * Portions Copyright (c) 1996-2008, PostgreSQL Global Development Group
10 * $PostgreSQL: pgsql/src/backend/tsearch/ts_typanalyze.c,v 1.3 2008/11/27 21:17:39 heikki Exp $
12 *-------------------------------------------------------------------------
16 #include "access/hash.h"
17 #include "catalog/pg_operator.h"
18 #include "commands/vacuum.h"
19 #include "tsearch/ts_type.h"
20 #include "utils/builtins.h"
21 #include "utils/hsearch.h"
24 /* A hash key for lexemes */
27 char *lexeme; /* lexeme (not NULL terminated!) */
28 int length; /* its length in bytes */
31 /* A hash table entry for the Lossy Counting algorithm */
34 LexemeHashKey key; /* This is 'e' from the LC algorithm. */
35 int frequency; /* This is 'f'. */
36 int delta; /* And this is 'delta'. */
39 static void compute_tsvector_stats(VacAttrStats *stats,
40 AnalyzeAttrFetchFunc fetchfunc,
43 static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current);
44 static uint32 lexeme_hash(const void *key, Size keysize);
45 static int lexeme_match(const void *key1, const void *key2, Size keysize);
46 static int lexeme_compare(const void *key1, const void *key2);
47 static int trackitem_compare_frequencies_desc(const void *e1, const void *e2);
48 static int trackitem_compare_lexemes(const void *e1, const void *e2);
52 * ts_typanalyze -- a custom typanalyze function for tsvector columns
55 ts_typanalyze(PG_FUNCTION_ARGS)
57 VacAttrStats *stats = (VacAttrStats *) PG_GETARG_POINTER(0);
58 Form_pg_attribute attr = stats->attr;
60 /* If the attstattarget column is negative, use the default value */
61 /* NB: it is okay to scribble on stats->attr since it's a copy */
62 if (attr->attstattarget < 0)
63 attr->attstattarget = default_statistics_target;
65 stats->compute_stats = compute_tsvector_stats;
66 /* see comment about the choice of minrows from analyze.c */
67 stats->minrows = 300 * attr->attstattarget;
73 * compute_tsvector_stats() -- compute statistics for a tsvector column
75 * This functions computes statistics that are useful for determining @@
76 * operations' selectivity, along with the fraction of non-null rows and
79 * Instead of finding the most common values, as we do for most datatypes,
80 * we're looking for the most common lexemes. This is more useful, because
81 * there most probably won't be any two rows with the same tsvector and thus
82 * the notion of a MCV is a bit bogus with this datatype. With a list of the
83 * most common lexemes we can do a better job at figuring out @@ selectivity.
85 * For the same reasons we assume that tsvector columns are unique when
86 * determining the number of distinct values.
88 * The algorithm used is Lossy Counting, as proposed in the paper "Approximate
89 * frequency counts over data streams" by G. S. Manku and R. Motwani, in
90 * Proceedings of the 28th International Conference on Very Large Data Bases,
91 * Hong Kong, China, August 2002, section 4.2. The paper is available at
92 * http://www.vldb.org/conf/2002/S10P03.pdf
94 * The Lossy Counting (aka LC) algorithm goes like this:
95 * Let D be a set of triples (e, f, d), where e is an element value, f is
96 * that element's frequency (occurrence count) and d is the maximum error in
97 * f. We start with D empty and process the elements in batches of size
98 * w. (The batch size is also known as "bucket size".) Let the current batch
99 * number be b_current, starting with 1. For each element e we either
100 * increment its f count, if it's already in D, or insert a new triple into D
101 * with values (e, 1, b_current - 1). After processing each batch we prune D,
102 * by removing from it all elements with f + d <= b_current. Finally, we
103 * gather elements with largest f. The LC paper proves error bounds on f
104 * dependent on the batch size w, and shows that the required table size
105 * is no more than a few times w.
107 * We use a hashtable for the D structure and a bucket width of
108 * statistic_target * 100, where 100 is an arbitrarily chosen constant, meant
109 * to approximate the number of lexemes in a single tsvector.
112 compute_tsvector_stats(VacAttrStats *stats,
113 AnalyzeAttrFetchFunc fetchfunc,
119 double total_width = 0;
120 /* This is D from the LC algorithm. */
123 HASH_SEQ_STATUS scan_status;
124 /* This is the current bucket number from the LC algorithm */
126 /* This is 'w' from the LC algorithm */
130 LexemeHashKey hash_key;
133 /* We want statistic_target * 100 lexemes in the MCELEM array */
134 num_mcelem = stats->attr->attstattarget * 100;
137 * We set bucket width equal to the target number of result lexemes.
138 * This is probably about right but perhaps might need to be scaled
141 bucket_width = num_mcelem;
144 * Create the hashtable. It will be in local memory, so we don't need to
145 * worry about initial size too much. Also we don't need to pay any
146 * attention to locking and memory management.
148 MemSet(&hash_ctl, 0, sizeof(hash_ctl));
149 hash_ctl.keysize = sizeof(LexemeHashKey);
150 hash_ctl.entrysize = sizeof(TrackItem);
151 hash_ctl.hash = lexeme_hash;
152 hash_ctl.match = lexeme_match;
153 hash_ctl.hcxt = CurrentMemoryContext;
154 lexemes_tab = hash_create("Analyzed lexemes table",
157 HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT);
159 /* Initialize counters. */
163 /* Loop over the tsvectors. */
164 for (vector_no = 0; vector_no < samplerows; vector_no++)
169 WordEntry *curentryptr;
173 vacuum_delay_point();
175 value = fetchfunc(stats, vector_no, &isnull);
178 * Check for null/nonnull.
187 * Add up widths for average-width calculation. Since it's a
188 * tsvector, we know it's varlena. As in the regular
189 * compute_minimal_stats function, we use the toasted width for this
192 total_width += VARSIZE_ANY(DatumGetPointer(value));
195 * Now detoast the tsvector if needed.
197 vector = DatumGetTSVector(value);
200 * We loop through the lexemes in the tsvector and add them to our
201 * tracking hashtable. Note: the hashtable entries will point into
202 * the (detoasted) tsvector value, therefore we cannot free that
203 * storage until we're done.
205 lexemesptr = STRPTR(vector);
206 curentryptr = ARRPTR(vector);
207 for (j = 0; j < vector->size; j++)
211 /* Construct a hash key */
212 hash_key.lexeme = lexemesptr + curentryptr->pos;
213 hash_key.length = curentryptr->len;
215 /* Lookup current lexeme in hashtable, adding it if new */
216 item = (TrackItem *) hash_search(lexemes_tab,
217 (const void *) &hash_key,
222 /* The lexeme is already on the tracking list */
227 /* Initialize new tracking list element */
229 item->delta = b_current - 1;
232 /* We prune the D structure after processing each bucket */
233 if (lexeme_no % bucket_width == 0)
235 prune_lexemes_hashtable(lexemes_tab, b_current);
239 /* Advance to the next WordEntry in the tsvector */
245 /* We can only compute real stats if we found some non-null values. */
246 if (null_cnt < samplerows)
248 int nonnull_cnt = samplerows - null_cnt;
250 TrackItem **sort_table;
252 int minfreq, maxfreq;
254 stats->stats_valid = true;
255 /* Do the simple null-frac and average width stats */
256 stats->stanullfrac = (double) null_cnt / (double) samplerows;
257 stats->stawidth = total_width / (double) nonnull_cnt;
259 /* Assume it's a unique column (see notes above) */
260 stats->stadistinct = -1.0;
263 * Determine the top-N lexemes by simply copying pointers from the
264 * hashtable into an array and applying qsort()
266 track_len = hash_get_num_entries(lexemes_tab);
268 sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * track_len);
270 hash_seq_init(&scan_status, lexemes_tab);
272 while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
274 sort_table[i++] = item;
276 Assert(i == track_len);
278 qsort(sort_table, track_len, sizeof(TrackItem *),
279 trackitem_compare_frequencies_desc);
281 /* Suppress any single-occurrence items */
282 while (track_len > 0)
284 if (sort_table[track_len-1]->frequency > 1)
289 /* Determine the number of most common lexemes to be stored */
290 if (num_mcelem > track_len)
291 num_mcelem = track_len;
293 /* Generate MCELEM slot entry */
296 MemoryContext old_context;
297 Datum *mcelem_values;
298 float4 *mcelem_freqs;
300 /* Grab the minimal and maximal frequencies that will get stored */
301 minfreq = sort_table[num_mcelem - 1]->frequency;
302 maxfreq = sort_table[0]->frequency;
305 * We want to store statistics sorted on the lexeme value using
306 * first length, then byte-for-byte comparison. The reason for
307 * doing length comparison first is that we don't care about the
308 * ordering so long as it's consistent, and comparing lengths first
309 * gives us a chance to avoid a strncmp() call.
311 * This is different from what we do with scalar statistics -- they
312 * get sorted on frequencies. The rationale is that we usually
313 * search through most common elements looking for a specific
314 * value, so we can grab its frequency. When values are presorted
315 * we can employ binary search for that. See ts_selfuncs.c for a
316 * real usage scenario.
318 qsort(sort_table, num_mcelem, sizeof(TrackItem *),
319 trackitem_compare_lexemes);
321 /* Must copy the target values into anl_context */
322 old_context = MemoryContextSwitchTo(stats->anl_context);
325 * We sorted statistics on the lexeme value, but we want to be
326 * able to find out the minimal and maximal frequency without
327 * going through all the values. We keep those two extra
328 * frequencies in two extra cells in mcelem_freqs.
330 mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
331 mcelem_freqs = (float4 *) palloc((num_mcelem + 2) * sizeof(float4));
333 for (i = 0; i < num_mcelem; i++)
335 TrackItem *item = sort_table[i];
338 PointerGetDatum(cstring_to_text_with_len(item->key.lexeme,
340 mcelem_freqs[i] = (double) item->frequency / (double) nonnull_cnt;
342 mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt;
343 mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt;
344 MemoryContextSwitchTo(old_context);
346 stats->stakind[0] = STATISTIC_KIND_MCELEM;
347 stats->staop[0] = TextEqualOperator;
348 stats->stanumbers[0] = mcelem_freqs;
349 /* See above comment about two extra frequency fields */
350 stats->numnumbers[0] = num_mcelem + 2;
351 stats->stavalues[0] = mcelem_values;
352 stats->numvalues[0] = num_mcelem;
353 /* We are storing text values */
354 stats->statypid[0] = TEXTOID;
355 stats->statyplen[0] = -1; /* typlen, -1 for varlena */
356 stats->statypbyval[0] = false;
357 stats->statypalign[0] = 'i';
362 /* We found only nulls; assume the column is entirely null */
363 stats->stats_valid = true;
364 stats->stanullfrac = 1.0;
365 stats->stawidth = 0; /* "unknown" */
366 stats->stadistinct = 0.0; /* "unknown" */
370 * We don't need to bother cleaning up any of our temporary palloc's.
371 * The hashtable should also go away, as it used a child memory context.
376 * A function to prune the D structure from the Lossy Counting algorithm.
377 * Consult compute_tsvector_stats() for wider explanation.
380 prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current)
382 HASH_SEQ_STATUS scan_status;
385 hash_seq_init(&scan_status, lexemes_tab);
386 while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
388 if (item->frequency + item->delta <= b_current)
390 if (hash_search(lexemes_tab, (const void *) &item->key,
391 HASH_REMOVE, NULL) == NULL)
392 elog(ERROR, "hash table corrupted");
398 * Hash functions for lexemes. They are strings, but not NULL terminated,
399 * so we need a special hash function.
402 lexeme_hash(const void *key, Size keysize)
404 const LexemeHashKey *l = (const LexemeHashKey *) key;
406 return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
411 * Matching function for lexemes, to be used in hashtable lookups.
414 lexeme_match(const void *key1, const void *key2, Size keysize)
416 /* The keysize parameter is superfluous, the keys store their lengths */
417 return lexeme_compare(key1, key2);
421 * Comparison function for lexemes.
424 lexeme_compare(const void *key1, const void *key2)
426 const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
427 const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
429 /* First, compare by length */
430 if (d1->length > d2->length)
432 else if (d1->length < d2->length)
434 /* Lengths are equal, do a byte-by-byte comparison */
435 return strncmp(d1->lexeme, d2->lexeme, d1->length);
439 * qsort() comparator for sorting TrackItems on frequencies (descending sort)
442 trackitem_compare_frequencies_desc(const void *e1, const void *e2)
444 const TrackItem * const *t1 = (const TrackItem * const *) e1;
445 const TrackItem * const *t2 = (const TrackItem * const *) e2;
447 return (*t2)->frequency - (*t1)->frequency;
451 * qsort() comparator for sorting TrackItems on lexemes
454 trackitem_compare_lexemes(const void *e1, const void *e2)
456 const TrackItem * const *t1 = (const TrackItem * const *) e1;
457 const TrackItem * const *t2 = (const TrackItem * const *) e2;
459 return lexeme_compare(&(*t1)->key, &(*t2)->key);