4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
23 * This file represents the QAT implementation of checksums and encryption.
24 * Internally, QAT shares the same cryptographic instances for both of these
25 * operations, so the code has been combined here. QAT data compression uses
26 * compression instances, so that code is separated into qat_compress.c
29 #if defined(_KERNEL) && defined(HAVE_QAT)
30 #include <linux/slab.h>
31 #include <linux/vmalloc.h>
32 #include <linux/pagemap.h>
33 #include <linux/completion.h>
34 #include <sys/zfs_context.h>
35 #include <sys/zio_crypt.h>
36 #include "lac/cpa_cy_im.h"
37 #include "lac/cpa_cy_common.h"
41 * Max instances in a QAT device, each instance is a channel to submit
42 * jobs to QAT hardware, this is only for pre-allocating instances
43 * and session arrays; the actual number of instances are defined in
44 * the QAT driver's configure file.
46 #define QAT_CRYPT_MAX_INSTANCES 48
48 #define MAX_PAGE_NUM 1024
50 static Cpa32U inst_num = 0;
51 static Cpa16U num_inst = 0;
52 static CpaInstanceHandle cy_inst_handles[QAT_CRYPT_MAX_INSTANCES];
53 static boolean_t qat_crypt_init_done = B_FALSE;
54 int zfs_qat_encrypt_disable = 0;
55 int zfs_qat_checksum_disable = 0;
57 typedef struct cy_callback {
58 CpaBoolean verify_result;
59 struct completion complete;
63 symcallback(void *p_callback, CpaStatus status, const CpaCySymOp operation,
64 void *op_data, CpaBufferList *buf_list_dst, CpaBoolean verify)
66 cy_callback_t *cb = p_callback;
69 /* indicate that the function has been called */
70 cb->verify_result = verify;
71 complete(&cb->complete);
76 qat_crypt_use_accel(size_t s_len)
78 return (!zfs_qat_encrypt_disable &&
79 qat_crypt_init_done &&
80 s_len >= QAT_MIN_BUF_SIZE &&
81 s_len <= QAT_MAX_BUF_SIZE);
85 qat_checksum_use_accel(size_t s_len)
87 return (!zfs_qat_checksum_disable &&
88 qat_crypt_init_done &&
89 s_len >= QAT_MIN_BUF_SIZE &&
90 s_len <= QAT_MAX_BUF_SIZE);
96 for (Cpa16U i = 0; i < num_inst; i++)
97 cpaCyStopInstance(cy_inst_handles[i]);
100 qat_crypt_init_done = B_FALSE;
106 CpaStatus status = CPA_STATUS_FAIL;
108 status = cpaCyGetNumInstances(&num_inst);
109 if (status != CPA_STATUS_SUCCESS)
112 /* if the user has configured no QAT encryption units just return */
116 if (num_inst > QAT_CRYPT_MAX_INSTANCES)
117 num_inst = QAT_CRYPT_MAX_INSTANCES;
119 status = cpaCyGetInstances(num_inst, &cy_inst_handles[0]);
120 if (status != CPA_STATUS_SUCCESS)
123 for (Cpa16U i = 0; i < num_inst; i++) {
124 status = cpaCySetAddressTranslation(cy_inst_handles[i],
125 (void *)virt_to_phys);
126 if (status != CPA_STATUS_SUCCESS)
129 status = cpaCyStartInstance(cy_inst_handles[i]);
130 if (status != CPA_STATUS_SUCCESS)
134 qat_crypt_init_done = B_TRUE;
145 if (!qat_crypt_init_done)
152 qat_init_crypt_session_ctx(qat_encrypt_dir_t dir, CpaInstanceHandle inst_handle,
153 CpaCySymSessionCtx **cy_session_ctx, crypto_key_t *key,
154 Cpa64U crypt, Cpa32U aad_len)
156 CpaStatus status = CPA_STATUS_SUCCESS;
158 Cpa32U ciper_algorithm;
159 Cpa32U hash_algorithm;
160 CpaCySymSessionSetupData sd = { 0 };
162 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_CCM) {
163 return (CPA_STATUS_FAIL);
165 ciper_algorithm = CPA_CY_SYM_CIPHER_AES_GCM;
166 hash_algorithm = CPA_CY_SYM_HASH_AES_GCM;
169 sd.cipherSetupData.cipherAlgorithm = ciper_algorithm;
170 sd.cipherSetupData.pCipherKey = key->ck_data;
171 sd.cipherSetupData.cipherKeyLenInBytes = key->ck_length / 8;
172 sd.hashSetupData.hashAlgorithm = hash_algorithm;
173 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_AUTH;
174 sd.hashSetupData.digestResultLenInBytes = ZIO_DATA_MAC_LEN;
175 sd.hashSetupData.authModeSetupData.aadLenInBytes = aad_len;
176 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
177 sd.symOperation = CPA_CY_SYM_OP_ALGORITHM_CHAINING;
178 sd.digestIsAppended = CPA_FALSE;
179 sd.verifyDigest = CPA_FALSE;
181 if (dir == QAT_ENCRYPT) {
182 sd.cipherSetupData.cipherDirection =
183 CPA_CY_SYM_CIPHER_DIRECTION_ENCRYPT;
185 CPA_CY_SYM_ALG_CHAIN_ORDER_HASH_THEN_CIPHER;
187 ASSERT3U(dir, ==, QAT_DECRYPT);
188 sd.cipherSetupData.cipherDirection =
189 CPA_CY_SYM_CIPHER_DIRECTION_DECRYPT;
191 CPA_CY_SYM_ALG_CHAIN_ORDER_CIPHER_THEN_HASH;
194 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
195 if (status != CPA_STATUS_SUCCESS)
198 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
199 if (status != CPA_STATUS_SUCCESS)
202 status = cpaCySymInitSession(inst_handle, symcallback, &sd,
204 if (status != CPA_STATUS_SUCCESS) {
205 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
209 return (CPA_STATUS_SUCCESS);
213 qat_init_checksum_session_ctx(CpaInstanceHandle inst_handle,
214 CpaCySymSessionCtx **cy_session_ctx, Cpa64U cksum)
216 CpaStatus status = CPA_STATUS_SUCCESS;
218 Cpa32U hash_algorithm;
219 CpaCySymSessionSetupData sd = { 0 };
222 * ZFS's SHA512 checksum is actually SHA512/256, which uses
223 * a different IV from standard SHA512. QAT does not support
224 * SHA512/256, so we can only support SHA256.
226 if (cksum == ZIO_CHECKSUM_SHA256)
227 hash_algorithm = CPA_CY_SYM_HASH_SHA256;
229 return (CPA_STATUS_FAIL);
231 sd.sessionPriority = CPA_CY_PRIORITY_NORMAL;
232 sd.symOperation = CPA_CY_SYM_OP_HASH;
233 sd.hashSetupData.hashAlgorithm = hash_algorithm;
234 sd.hashSetupData.hashMode = CPA_CY_SYM_HASH_MODE_PLAIN;
235 sd.hashSetupData.digestResultLenInBytes = sizeof (zio_cksum_t);
236 sd.digestIsAppended = CPA_FALSE;
237 sd.verifyDigest = CPA_FALSE;
239 status = cpaCySymSessionCtxGetSize(inst_handle, &sd, &ctx_size);
240 if (status != CPA_STATUS_SUCCESS)
243 status = QAT_PHYS_CONTIG_ALLOC(cy_session_ctx, ctx_size);
244 if (status != CPA_STATUS_SUCCESS)
247 status = cpaCySymInitSession(inst_handle, symcallback, &sd,
249 if (status != CPA_STATUS_SUCCESS) {
250 QAT_PHYS_CONTIG_FREE(*cy_session_ctx);
254 return (CPA_STATUS_SUCCESS);
258 qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs,
259 CpaBufferList *src, CpaBufferList *dst)
261 CpaStatus status = CPA_STATUS_SUCCESS;
262 Cpa32U meta_size = 0;
264 status = cpaCyBufferListGetMetaSize(inst_handle, nr_bufs, &meta_size);
265 if (status != CPA_STATUS_SUCCESS)
268 status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size);
269 if (status != CPA_STATUS_SUCCESS)
273 status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData,
275 if (status != CPA_STATUS_SUCCESS)
279 return (CPA_STATUS_SUCCESS);
282 QAT_PHYS_CONTIG_FREE(src->pPrivateMetaData);
284 QAT_PHYS_CONTIG_FREE(dst->pPrivateMetaData);
290 qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf,
291 uint8_t *aad_buf, uint32_t aad_len, uint8_t *iv_buf, uint8_t *digest_buf,
292 crypto_key_t *key, uint64_t crypt, uint32_t enc_len)
294 CpaStatus status = CPA_STATUS_SUCCESS;
296 CpaInstanceHandle cy_inst_handle;
297 Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2;
298 Cpa32U bytes_left = 0;
300 CpaCySymSessionCtx *cy_session_ctx = NULL;
302 CpaCySymOpData op_data = { 0 };
303 CpaBufferList src_buffer_list = { 0 };
304 CpaBufferList dst_buffer_list = { 0 };
305 CpaFlatBuffer *flat_src_buf_array = NULL;
306 CpaFlatBuffer *flat_src_buf = NULL;
307 CpaFlatBuffer *flat_dst_buf_array = NULL;
308 CpaFlatBuffer *flat_dst_buf = NULL;
309 struct page *in_pages[MAX_PAGE_NUM];
310 struct page *out_pages[MAX_PAGE_NUM];
311 Cpa32U in_page_num = 0;
312 Cpa32U out_page_num = 0;
313 Cpa32U in_page_off = 0;
314 Cpa32U out_page_off = 0;
316 if (dir == QAT_ENCRYPT) {
317 QAT_STAT_BUMP(encrypt_requests);
318 QAT_STAT_INCR(encrypt_total_in_bytes, enc_len);
320 QAT_STAT_BUMP(decrypt_requests);
321 QAT_STAT_INCR(decrypt_total_in_bytes, enc_len);
324 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
325 cy_inst_handle = cy_inst_handles[i];
327 status = qat_init_crypt_session_ctx(dir, cy_inst_handle,
328 &cy_session_ctx, key, crypt, aad_len);
329 if (status != CPA_STATUS_SUCCESS) {
330 /* don't count CCM as a failure since it's not supported */
331 if (zio_crypt_table[crypt].ci_crypt_type == ZC_TYPE_GCM)
332 QAT_STAT_BUMP(crypt_fails);
337 * We increment nr_bufs by 2 to allow us to handle non
338 * page-aligned buffer addresses and buffers whose sizes
339 * are not divisible by PAGE_SIZE.
341 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
342 &src_buffer_list, &dst_buffer_list);
343 if (status != CPA_STATUS_SUCCESS)
346 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
347 nr_bufs * sizeof (CpaFlatBuffer));
348 if (status != CPA_STATUS_SUCCESS)
350 status = QAT_PHYS_CONTIG_ALLOC(&flat_dst_buf_array,
351 nr_bufs * sizeof (CpaFlatBuffer));
352 if (status != CPA_STATUS_SUCCESS)
355 bytes_left = enc_len;
357 flat_src_buf = flat_src_buf_array;
358 while (bytes_left > 0) {
359 in_page_off = ((long)data & ~PAGE_MASK);
360 in_pages[in_page_num] = qat_mem_to_page(data);
361 flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off;
362 flat_src_buf->dataLenInBytes =
363 min((long)PAGE_SIZE - in_page_off, (long)bytes_left);
364 data += flat_src_buf->dataLenInBytes;
365 bytes_left -= flat_src_buf->dataLenInBytes;
369 src_buffer_list.pBuffers = flat_src_buf_array;
370 src_buffer_list.numBuffers = in_page_num;
372 bytes_left = enc_len;
374 flat_dst_buf = flat_dst_buf_array;
375 while (bytes_left > 0) {
376 out_page_off = ((long)data & ~PAGE_MASK);
377 out_pages[out_page_num] = qat_mem_to_page(data);
378 flat_dst_buf->pData = kmap(out_pages[out_page_num]) +
380 flat_dst_buf->dataLenInBytes =
381 min((long)PAGE_SIZE - out_page_off, (long)bytes_left);
382 data += flat_dst_buf->dataLenInBytes;
383 bytes_left -= flat_dst_buf->dataLenInBytes;
387 dst_buffer_list.pBuffers = flat_dst_buf_array;
388 dst_buffer_list.numBuffers = out_page_num;
390 op_data.sessionCtx = cy_session_ctx;
391 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
392 op_data.pIv = NULL; /* set this later as the J0 block */
393 op_data.ivLenInBytes = 0;
394 op_data.cryptoStartSrcOffsetInBytes = 0;
395 op_data.messageLenToCipherInBytes = 0;
396 op_data.hashStartSrcOffsetInBytes = 0;
397 op_data.messageLenToHashInBytes = 0;
398 op_data.pDigestResult = 0;
399 op_data.messageLenToCipherInBytes = enc_len;
400 op_data.ivLenInBytes = ZIO_DATA_IV_LEN;
401 op_data.pDigestResult = digest_buf;
402 op_data.pAdditionalAuthData = aad_buf;
403 op_data.pIv = iv_buf;
405 cb.verify_result = CPA_FALSE;
406 init_completion(&cb.complete);
407 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
408 &src_buffer_list, &dst_buffer_list, NULL);
409 if (status != CPA_STATUS_SUCCESS)
412 if (!wait_for_completion_interruptible_timeout(&cb.complete,
414 status = CPA_STATUS_FAIL;
418 if (cb.verify_result == CPA_FALSE) {
419 status = CPA_STATUS_FAIL;
423 if (dir == QAT_ENCRYPT)
424 QAT_STAT_INCR(encrypt_total_out_bytes, enc_len);
426 QAT_STAT_INCR(decrypt_total_out_bytes, enc_len);
429 if (status != CPA_STATUS_SUCCESS)
430 QAT_STAT_BUMP(crypt_fails);
432 for (i = 0; i < in_page_num; i++)
434 for (i = 0; i < out_page_num; i++)
435 kunmap(out_pages[i]);
437 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
438 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
439 QAT_PHYS_CONTIG_FREE(dst_buffer_list.pPrivateMetaData);
440 QAT_PHYS_CONTIG_FREE(cy_session_ctx);
441 QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
442 QAT_PHYS_CONTIG_FREE(flat_dst_buf_array);
448 qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp)
452 CpaInstanceHandle cy_inst_handle;
453 Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2;
454 Cpa32U bytes_left = 0;
456 CpaCySymSessionCtx *cy_session_ctx = NULL;
458 Cpa8U digest_buffer[sizeof (zio_cksum_t)];
459 CpaCySymOpData op_data = { 0 };
460 CpaBufferList src_buffer_list = { 0 };
461 CpaFlatBuffer *flat_src_buf_array = NULL;
462 CpaFlatBuffer *flat_src_buf = NULL;
463 struct page *in_pages[MAX_PAGE_NUM];
467 QAT_STAT_BUMP(cksum_requests);
468 QAT_STAT_INCR(cksum_total_in_bytes, size);
470 i = (Cpa32U)atomic_inc_32_nv(&inst_num) % num_inst;
471 cy_inst_handle = cy_inst_handles[i];
473 status = qat_init_checksum_session_ctx(cy_inst_handle,
474 &cy_session_ctx, cksum);
475 if (status != CPA_STATUS_SUCCESS) {
476 /* don't count unsupported checksums as a failure */
477 if (cksum == ZIO_CHECKSUM_SHA256 ||
478 cksum == ZIO_CHECKSUM_SHA512)
479 QAT_STAT_BUMP(cksum_fails);
484 * We increment nr_bufs by 2 to allow us to handle non
485 * page-aligned buffer addresses and buffers whose sizes
486 * are not divisible by PAGE_SIZE.
488 status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs,
489 &src_buffer_list, &src_buffer_list);
490 if (status != CPA_STATUS_SUCCESS)
493 status = QAT_PHYS_CONTIG_ALLOC(&flat_src_buf_array,
494 nr_bufs * sizeof (CpaFlatBuffer));
495 if (status != CPA_STATUS_SUCCESS)
500 flat_src_buf = flat_src_buf_array;
501 while (bytes_left > 0) {
502 page_off = ((long)data & ~PAGE_MASK);
503 in_pages[page_num] = qat_mem_to_page(data);
504 flat_src_buf->pData = kmap(in_pages[page_num]) + page_off;
505 flat_src_buf->dataLenInBytes =
506 min((long)PAGE_SIZE - page_off, (long)bytes_left);
507 data += flat_src_buf->dataLenInBytes;
508 bytes_left -= flat_src_buf->dataLenInBytes;
512 src_buffer_list.pBuffers = flat_src_buf_array;
513 src_buffer_list.numBuffers = page_num;
515 op_data.sessionCtx = cy_session_ctx;
516 op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;
517 op_data.hashStartSrcOffsetInBytes = 0;
518 op_data.messageLenToHashInBytes = size;
519 op_data.pDigestResult = digest_buffer;
521 cb.verify_result = CPA_FALSE;
522 init_completion(&cb.complete);
523 status = cpaCySymPerformOp(cy_inst_handle, &cb, &op_data,
524 &src_buffer_list, &src_buffer_list, NULL);
525 if (status != CPA_STATUS_SUCCESS)
528 if (!wait_for_completion_interruptible_timeout(&cb.complete,
530 status = CPA_STATUS_FAIL;
534 bcopy(digest_buffer, zcp, sizeof (zio_cksum_t));
537 if (status != CPA_STATUS_SUCCESS)
538 QAT_STAT_BUMP(cksum_fails);
540 for (i = 0; i < page_num; i++)
543 cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx);
544 QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData);
545 QAT_PHYS_CONTIG_FREE(cy_session_ctx);
546 QAT_PHYS_CONTIG_FREE(flat_src_buf_array);
551 module_param(zfs_qat_encrypt_disable, int, 0644);
552 MODULE_PARM_DESC(zfs_qat_encrypt_disable, "Disable QAT encryption");
554 module_param(zfs_qat_checksum_disable, int, 0644);
555 MODULE_PARM_DESC(zfs_qat_checksum_disable, "Disable QAT checksumming");