granicus.if.org Git - zfs/blob - module/zfs/qat_compress.c

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21
  22 #if defined(_KERNEL) && defined(HAVE_QAT)
  23 #include <linux/slab.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/pagemap.h>
  26 #include <linux/completion.h>
  27 #include <sys/zfs_context.h>
  28 #include "qat_compress.h"
  29
  30 /*
  31  * Timeout - no response from hardware after 0.5 seconds
  32  */
  33 #define TIMEOUT_MS              500
  34
  35 /*
  36  * Max instances in QAT device, each instance is a channel to submit
  37  * jobs to QAT hardware, this is only for pre-allocating instance,
  38  * and session arrays, the actual number of instances are defined in
  39  * the QAT driver's configure file.
  40  */
  41 #define MAX_INSTANCES           48
  42
  43 /*
  44  * ZLIB head and foot size
  45  */
  46 #define ZLIB_HEAD_SZ            2
  47 #define ZLIB_FOOT_SZ            4
  48
  49 /*
  50  * The minimal and maximal buffer size, which are not restricted
  51  * in the QAT hardware, but with the input buffer size between 4KB
  52  * and 128KB, the hardware can provide the optimal performance.
  53  */
  54 #define QAT_MIN_BUF_SIZE        (4*1024)
  55 #define QAT_MAX_BUF_SIZE        (128*1024)
  56
  57 /*
  58  * Used for qat kstat.
  59  */
  60 typedef struct qat_stats {
  61         /*
  62          * Number of jobs submitted to qat compression engine.
  63          */
  64         kstat_named_t comp_requests;
  65         /*
  66          * Total bytes sent to qat compression engine.
  67          */
  68         kstat_named_t comp_total_in_bytes;
  69         /*
  70          * Total bytes output from qat compression engine.
  71          */
  72         kstat_named_t comp_total_out_bytes;
  73         /*
  74          * Number of jobs submitted to qat de-compression engine.
  75          */
  76         kstat_named_t decomp_requests;
  77         /*
  78          * Total bytes sent to qat de-compression engine.
  79          */
  80         kstat_named_t decomp_total_in_bytes;
  81         /*
  82          * Total bytes output from qat de-compression engine.
  83          */
  84         kstat_named_t decomp_total_out_bytes;
  85         /*
  86          * Number of fails in qat engine.
  87          * Note: when qat fail happens, it doesn't mean a critical hardware
  88          * issue, sometimes it is because the output buffer is not big enough,
  89          * and the compression job will be transfered to gzip software again,
  90          * so the functionality of ZFS is not impacted.
  91          */
  92         kstat_named_t dc_fails;
  93 } qat_stats_t;
  94
  95 qat_stats_t qat_stats = {
  96         { "comp_reqests",                       KSTAT_DATA_UINT64 },
  97         { "comp_total_in_bytes",                KSTAT_DATA_UINT64 },
  98         { "comp_total_out_bytes",               KSTAT_DATA_UINT64 },
  99         { "decomp_reqests",                     KSTAT_DATA_UINT64 },
 100         { "decomp_total_in_bytes",              KSTAT_DATA_UINT64 },
 101         { "decomp_total_out_bytes",             KSTAT_DATA_UINT64 },
 102         { "dc_fails",                           KSTAT_DATA_UINT64 },
 103 };
 104
 105 static kstat_t *qat_ksp;
 106 static CpaInstanceHandle dc_inst_handles[MAX_INSTANCES];
 107 static CpaDcSessionHandle session_handles[MAX_INSTANCES];
 108 static CpaBufferList **buffer_array[MAX_INSTANCES];
 109 static Cpa16U num_inst = 0;
 110 static Cpa32U inst_num = 0;
 111 static boolean_t qat_init_done = B_FALSE;
 112 int zfs_qat_disable = 0;
 113
 114 #define QAT_STAT_INCR(stat, val) \
 115         atomic_add_64(&qat_stats.stat.value.ui64, (val));
 116 #define QAT_STAT_BUMP(stat) \
 117         QAT_STAT_INCR(stat, 1);
 118
 119 #define PHYS_CONTIG_ALLOC(pp_mem_addr, size_bytes)      \
 120         mem_alloc_contig((void *)(pp_mem_addr), (size_bytes))
 121
 122 #define PHYS_CONTIG_FREE(p_mem_addr)    \
 123         mem_free_contig((void *)&(p_mem_addr))
 124
 125 static inline struct page *
 126 mem_to_page(void *addr)
 127 {
 128         if (!is_vmalloc_addr(addr))
 129                 return (virt_to_page(addr));
 130
 131         return (vmalloc_to_page(addr));
 132 }
 133
 134 static void
 135 qat_dc_callback(void *p_callback, CpaStatus status)
 136 {
 137         if (p_callback != NULL)
 138                 complete((struct completion *)p_callback);
 139 }
 140
 141 static inline CpaStatus
 142 mem_alloc_contig(void **pp_mem_addr, Cpa32U size_bytes)
 143 {
 144         *pp_mem_addr = kmalloc(size_bytes, GFP_KERNEL);
 145         if (*pp_mem_addr == NULL)
 146                 return (CPA_STATUS_RESOURCE);
 147         return (CPA_STATUS_SUCCESS);
 148 }
 149
 150 static inline void
 151 mem_free_contig(void **pp_mem_addr)
 152 {
 153         if (*pp_mem_addr != NULL) {
 154                 kfree(*pp_mem_addr);
 155                 *pp_mem_addr = NULL;
 156         }
 157 }
 158
 159 static void
 160 qat_clean(void)
 161 {
 162         Cpa16U buff_num = 0;
 163         Cpa16U num_inter_buff_lists = 0;
 164         Cpa16U i = 0;
 165
 166         for (i = 0; i < num_inst; i++) {
 167                 cpaDcStopInstance(dc_inst_handles[i]);
 168                 PHYS_CONTIG_FREE(session_handles[i]);
 169                 /* free intermediate buffers  */
 170                 if (buffer_array[i] != NULL) {
 171                         cpaDcGetNumIntermediateBuffers(
 172                             dc_inst_handles[i], &num_inter_buff_lists);
 173                         for (buff_num = 0; buff_num < num_inter_buff_lists;
 174                             buff_num++) {
 175                                 CpaBufferList *buffer_inter =
 176                                     buffer_array[i][buff_num];
 177                                 if (buffer_inter->pBuffers) {
 178                                         PHYS_CONTIG_FREE(
 179                                             buffer_inter->pBuffers->pData);
 180                                         PHYS_CONTIG_FREE(
 181                                             buffer_inter->pBuffers);
 182                                 }
 183                                 PHYS_CONTIG_FREE(
 184                                     buffer_inter->pPrivateMetaData);
 185                                 PHYS_CONTIG_FREE(buffer_inter);
 186                         }
 187                 }
 188         }
 189
 190         num_inst = 0;
 191         qat_init_done = B_FALSE;
 192 }
 193
 194 int
 195 qat_init(void)
 196 {
 197         CpaStatus status = CPA_STATUS_SUCCESS;
 198         Cpa32U sess_size = 0;
 199         Cpa32U ctx_size = 0;
 200         Cpa16U num_inter_buff_lists = 0;
 201         Cpa16U buff_num = 0;
 202         Cpa32U buff_meta_size = 0;
 203         CpaDcSessionSetupData sd = {0};
 204         Cpa16U i;
 205
 206         status = cpaDcGetNumInstances(&num_inst);
 207         if (status != CPA_STATUS_SUCCESS || num_inst == 0)
 208                 return (-1);
 209
 210         if (num_inst > MAX_INSTANCES)
 211                 num_inst = MAX_INSTANCES;
 212
 213         status = cpaDcGetInstances(num_inst, &dc_inst_handles[0]);
 214         if (status != CPA_STATUS_SUCCESS)
 215                 return (-1);
 216
 217         for (i = 0; i < num_inst; i++) {
 218                 cpaDcSetAddressTranslation(dc_inst_handles[i],
 219                     (void*)virt_to_phys);
 220
 221                 status = cpaDcBufferListGetMetaSize(dc_inst_handles[i],
 222                     1, &buff_meta_size);
 223
 224                 if (status == CPA_STATUS_SUCCESS)
 225                         status = cpaDcGetNumIntermediateBuffers(
 226                             dc_inst_handles[i], &num_inter_buff_lists);
 227
 228                 if (status == CPA_STATUS_SUCCESS && num_inter_buff_lists != 0)
 229                         status = PHYS_CONTIG_ALLOC(&buffer_array[i],
 230                             num_inter_buff_lists *
 231                             sizeof (CpaBufferList *));
 232
 233                 for (buff_num = 0; buff_num < num_inter_buff_lists;
 234                     buff_num++) {
 235                         if (status == CPA_STATUS_SUCCESS)
 236                                 status = PHYS_CONTIG_ALLOC(
 237                                     &buffer_array[i][buff_num],
 238                                     sizeof (CpaBufferList));
 239
 240                         if (status == CPA_STATUS_SUCCESS)
 241                                 status = PHYS_CONTIG_ALLOC(
 242                                     &buffer_array[i][buff_num]->
 243                                     pPrivateMetaData,
 244                                     buff_meta_size);
 245
 246                         if (status == CPA_STATUS_SUCCESS)
 247                                 status = PHYS_CONTIG_ALLOC(
 248                                     &buffer_array[i][buff_num]->pBuffers,
 249                                     sizeof (CpaFlatBuffer));
 250
 251                         if (status == CPA_STATUS_SUCCESS) {
 252                                 /*
 253                                  *  implementation requires an intermediate
 254                                  *  buffer approximately twice the size of
 255                                  *  output buffer, which is 2x max buffer
 256                                  *  size here.
 257                                  */
 258                                 status = PHYS_CONTIG_ALLOC(
 259                                     &buffer_array[i][buff_num]->pBuffers->
 260                                     pData, 2 * QAT_MAX_BUF_SIZE);
 261                                 if (status != CPA_STATUS_SUCCESS)
 262                                         goto fail;
 263
 264                                 buffer_array[i][buff_num]->numBuffers = 1;
 265                                 buffer_array[i][buff_num]->pBuffers->
 266                                     dataLenInBytes = 2 * QAT_MAX_BUF_SIZE;
 267                         }
 268                 }
 269
 270                 status = cpaDcStartInstance(dc_inst_handles[i],
 271                     num_inter_buff_lists, buffer_array[i]);
 272                 if (status != CPA_STATUS_SUCCESS)
 273                         goto fail;
 274
 275                 sd.compLevel = CPA_DC_L1;
 276                 sd.compType = CPA_DC_DEFLATE;
 277                 sd.huffType = CPA_DC_HT_FULL_DYNAMIC;
 278                 sd.sessDirection = CPA_DC_DIR_COMBINED;
 279                 sd.sessState = CPA_DC_STATELESS;
 280                 sd.deflateWindowSize = 7;
 281                 sd.checksum = CPA_DC_ADLER32;
 282                 status = cpaDcGetSessionSize(dc_inst_handles[i],
 283                     &sd, &sess_size, &ctx_size);
 284                 if (status != CPA_STATUS_SUCCESS)
 285                         goto fail;
 286
 287                 PHYS_CONTIG_ALLOC(&session_handles[i], sess_size);
 288                 if (session_handles[i] == NULL)
 289                         goto fail;
 290
 291                 status = cpaDcInitSession(dc_inst_handles[i],
 292                     session_handles[i],
 293                     &sd, NULL, qat_dc_callback);
 294                 if (status != CPA_STATUS_SUCCESS)
 295                         goto fail;
 296         }
 297
 298         qat_ksp = kstat_create("zfs", 0, "qat", "misc",
 299             KSTAT_TYPE_NAMED, sizeof (qat_stats) / sizeof (kstat_named_t),
 300             KSTAT_FLAG_VIRTUAL);
 301         if (qat_ksp != NULL) {
 302                 qat_ksp->ks_data = &qat_stats;
 303                 kstat_install(qat_ksp);
 304         }
 305
 306         qat_init_done = B_TRUE;
 307         return (0);
 308 fail:
 309         qat_clean();
 310         return (-1);
 311 }
 312
 313 void
 314 qat_fini(void)
 315 {
 316         qat_clean();
 317
 318         if (qat_ksp != NULL) {
 319                 kstat_delete(qat_ksp);
 320                 qat_ksp = NULL;
 321         }
 322 }
 323
 324 boolean_t
 325 qat_use_accel(size_t s_len)
 326 {
 327         return (!zfs_qat_disable &&
 328             qat_init_done &&
 329             s_len >= QAT_MIN_BUF_SIZE &&
 330             s_len <= QAT_MAX_BUF_SIZE);
 331 }
 332
 333 int
 334 qat_compress(qat_compress_dir_t dir, char *src, int src_len,
 335     char *dst, int dst_len, size_t *c_len)
 336 {
 337         CpaInstanceHandle dc_inst_handle;
 338         CpaDcSessionHandle session_handle;
 339         CpaBufferList *buf_list_src = NULL;
 340         CpaBufferList *buf_list_dst = NULL;
 341         CpaFlatBuffer *flat_buf_src = NULL;
 342         CpaFlatBuffer *flat_buf_dst = NULL;
 343         Cpa8U *buffer_meta_src = NULL;
 344         Cpa8U *buffer_meta_dst = NULL;
 345         Cpa32U buffer_meta_size = 0;
 346         CpaDcRqResults dc_results;
 347         CpaStatus status = CPA_STATUS_SUCCESS;
 348         Cpa32U hdr_sz = 0;
 349         Cpa32U compressed_sz;
 350         Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1;
 351         Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1;
 352         Cpa32U bytes_left;
 353         char *data;
 354         struct page *in_page, *out_page;
 355         struct page **in_pages = NULL;
 356         struct page **out_pages = NULL;
 357         struct completion complete;
 358         size_t ret = -1;
 359         Cpa16U page_num = 0;
 360         Cpa16U i;
 361
 362         Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) +
 363             (num_src_buf * sizeof (CpaFlatBuffer));
 364         Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) +
 365             (num_dst_buf * sizeof (CpaFlatBuffer));
 366
 367         if (!is_vmalloc_addr(src) || !is_vmalloc_addr(src + src_len - 1) ||
 368             !is_vmalloc_addr(dst) || !is_vmalloc_addr(dst + dst_len - 1))
 369                 return (-1);
 370
 371         if (PHYS_CONTIG_ALLOC(&in_pages,
 372             num_src_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
 373                 goto fail;
 374
 375         if (PHYS_CONTIG_ALLOC(&out_pages,
 376             num_dst_buf * sizeof (struct page *)) != CPA_STATUS_SUCCESS)
 377                 goto fail;
 378
 379         i = atomic_inc_32_nv(&inst_num) % num_inst;
 380         dc_inst_handle = dc_inst_handles[i];
 381         session_handle = session_handles[i];
 382
 383         cpaDcBufferListGetMetaSize(dc_inst_handle, num_src_buf,
 384             &buffer_meta_size);
 385         if (PHYS_CONTIG_ALLOC(&buffer_meta_src, buffer_meta_size) !=
 386             CPA_STATUS_SUCCESS)
 387                 goto fail;
 388
 389         cpaDcBufferListGetMetaSize(dc_inst_handle, num_dst_buf,
 390             &buffer_meta_size);
 391         if (PHYS_CONTIG_ALLOC(&buffer_meta_dst, buffer_meta_size) !=
 392             CPA_STATUS_SUCCESS)
 393                 goto fail;
 394
 395         /* build source buffer list */
 396         if (PHYS_CONTIG_ALLOC(&buf_list_src, src_buffer_list_mem_size) !=
 397             CPA_STATUS_SUCCESS)
 398                 goto fail;
 399
 400         flat_buf_src = (CpaFlatBuffer *)(buf_list_src + 1);
 401
 402         buf_list_src->pBuffers = flat_buf_src; /* always point to first one */
 403
 404         /* build destination buffer list */
 405         if (PHYS_CONTIG_ALLOC(&buf_list_dst, dst_buffer_list_mem_size) !=
 406             CPA_STATUS_SUCCESS)
 407                 goto fail;
 408
 409         flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
 410
 411         buf_list_dst->pBuffers = flat_buf_dst; /* always point to first one */
 412
 413         buf_list_src->numBuffers = 0;
 414         buf_list_src->pPrivateMetaData = buffer_meta_src;
 415         bytes_left = src_len;
 416         data = src;
 417         page_num = 0;
 418         while (bytes_left > 0) {
 419                 in_page = mem_to_page(data);
 420                 in_pages[page_num] = in_page;
 421                 flat_buf_src->pData = kmap(in_page);
 422                 flat_buf_src->dataLenInBytes =
 423                     min((long)bytes_left, (long)PAGE_SIZE);
 424
 425                 bytes_left -= flat_buf_src->dataLenInBytes;
 426                 data += flat_buf_src->dataLenInBytes;
 427                 flat_buf_src++;
 428                 buf_list_src->numBuffers++;
 429                 page_num++;
 430         }
 431
 432         buf_list_dst->numBuffers = 0;
 433         buf_list_dst->pPrivateMetaData = buffer_meta_dst;
 434         bytes_left = dst_len;
 435         data = dst;
 436         page_num = 0;
 437         while (bytes_left > 0) {
 438                 out_page = mem_to_page(data);
 439                 flat_buf_dst->pData = kmap(out_page);
 440                 out_pages[page_num] = out_page;
 441                 flat_buf_dst->dataLenInBytes =
 442                     min((long)bytes_left, (long)PAGE_SIZE);
 443
 444                 bytes_left -= flat_buf_dst->dataLenInBytes;
 445                 data += flat_buf_dst->dataLenInBytes;
 446                 flat_buf_dst++;
 447                 buf_list_dst->numBuffers++;
 448                 page_num++;
 449         }
 450
 451         init_completion(&complete);
 452
 453         if (dir == QAT_COMPRESS) {
 454                 QAT_STAT_BUMP(comp_requests);
 455                 QAT_STAT_INCR(comp_total_in_bytes, src_len);
 456
 457                 cpaDcGenerateHeader(session_handle,
 458                     buf_list_dst->pBuffers, &hdr_sz);
 459                 buf_list_dst->pBuffers->pData += hdr_sz;
 460                 buf_list_dst->pBuffers->dataLenInBytes -= hdr_sz;
 461                 status = cpaDcCompressData(
 462                     dc_inst_handle, session_handle,
 463                     buf_list_src, buf_list_dst,
 464                     &dc_results, CPA_DC_FLUSH_FINAL,
 465                     &complete);
 466                 if (status != CPA_STATUS_SUCCESS) {
 467                         goto fail;
 468                 }
 469
 470                 /* we now wait until the completion of the operation. */
 471                 if (!wait_for_completion_interruptible_timeout(&complete,
 472                     TIMEOUT_MS)) {
 473                         status = CPA_STATUS_FAIL;
 474                         goto fail;
 475                 }
 476
 477                 if (dc_results.status != CPA_STATUS_SUCCESS) {
 478                         status = CPA_STATUS_FAIL;
 479                         goto fail;
 480                 }
 481
 482                 compressed_sz = dc_results.produced;
 483                 if (compressed_sz + hdr_sz + ZLIB_FOOT_SZ > dst_len) {
 484                         goto fail;
 485                 }
 486
 487                 flat_buf_dst = (CpaFlatBuffer *)(buf_list_dst + 1);
 488                 /* move to the last page */
 489                 flat_buf_dst += (compressed_sz + hdr_sz) >> PAGE_SHIFT;
 490
 491                 /* no space for gzip foot in the last page */
 492                 if (((compressed_sz + hdr_sz) % PAGE_SIZE)
 493                     + ZLIB_FOOT_SZ > PAGE_SIZE)
 494                         goto fail;
 495
 496                 flat_buf_dst->pData += (compressed_sz + hdr_sz) % PAGE_SIZE;
 497                 flat_buf_dst->dataLenInBytes = ZLIB_FOOT_SZ;
 498
 499                 dc_results.produced = 0;
 500                 status = cpaDcGenerateFooter(session_handle,
 501                     flat_buf_dst, &dc_results);
 502                 if (status != CPA_STATUS_SUCCESS) {
 503                         goto fail;
 504                 }
 505
 506                 *c_len = compressed_sz + dc_results.produced + hdr_sz;
 507
 508                 if (*c_len < PAGE_SIZE)
 509                         *c_len = 8 * PAGE_SIZE;
 510
 511                 QAT_STAT_INCR(comp_total_out_bytes, *c_len);
 512
 513                 ret = 0;
 514
 515         } else if (dir == QAT_DECOMPRESS) {
 516                 QAT_STAT_BUMP(decomp_requests);
 517                 QAT_STAT_INCR(decomp_total_in_bytes, src_len);
 518
 519                 buf_list_src->pBuffers->pData += ZLIB_HEAD_SZ;
 520                 buf_list_src->pBuffers->dataLenInBytes -= ZLIB_HEAD_SZ;
 521                 status = cpaDcDecompressData(dc_inst_handle,
 522                     session_handle,
 523                     buf_list_src,
 524                     buf_list_dst,
 525                     &dc_results,
 526                     CPA_DC_FLUSH_FINAL,
 527                     &complete);
 528
 529                 if (CPA_STATUS_SUCCESS != status) {
 530                         status = CPA_STATUS_FAIL;
 531                         goto fail;
 532                 }
 533
 534                 /* we now wait until the completion of the operation. */
 535                 if (!wait_for_completion_interruptible_timeout(&complete,
 536                     TIMEOUT_MS)) {
 537                         status = CPA_STATUS_FAIL;
 538                         goto fail;
 539                 }
 540
 541                 if (dc_results.status != CPA_STATUS_SUCCESS) {
 542                         status = CPA_STATUS_FAIL;
 543                         goto fail;
 544                 }
 545
 546                 *c_len = dc_results.produced;
 547
 548                 QAT_STAT_INCR(decomp_total_out_bytes, *c_len);
 549
 550                 ret = 0;
 551         }
 552
 553 fail:
 554         if (status != CPA_STATUS_SUCCESS) {
 555                 QAT_STAT_BUMP(dc_fails);
 556         }
 557
 558         if (in_pages) {
 559                 for (page_num = 0;
 560                     page_num < buf_list_src->numBuffers;
 561                     page_num++) {
 562                         kunmap(in_pages[page_num]);
 563                 }
 564                 PHYS_CONTIG_FREE(in_pages);
 565         }
 566
 567         if (out_pages) {
 568                 for (page_num = 0;
 569                     page_num < buf_list_dst->numBuffers;
 570                     page_num++) {
 571                         kunmap(out_pages[page_num]);
 572                 }
 573                 PHYS_CONTIG_FREE(out_pages);
 574         }
 575
 576         PHYS_CONTIG_FREE(buffer_meta_src);
 577         PHYS_CONTIG_FREE(buffer_meta_dst);
 578         PHYS_CONTIG_FREE(buf_list_src);
 579         PHYS_CONTIG_FREE(buf_list_dst);
 580
 581         return (ret);
 582 }
 583
 584 module_param(zfs_qat_disable, int, 0644);
 585 MODULE_PARM_DESC(zfs_qat_disable, "Disable QAT compression");
 586
 587 #endif