From: Tom Caputi Date: Fri, 16 Mar 2018 23:02:49 +0000 (-0400) Subject: Allow QAT to handle non page-aligned buffers X-Git-Tag: zfs-0.8.0-rc1~271 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=17dd88352e655afe1d09e8e75ca1415e79c3510a;p=zfs Allow QAT to handle non page-aligned buffers This patch adds some handling to the QAT acceleration functions that allows them to handle buffers that are not aligned with the page cache. At the moment this never happens since callers only happen to work with page-aligned buffers, but this code should prevent headaches if this isn't always true in the future. This patch also adds some cleanups to align the QAT compression code with the encryption and checksumming code. Reviewed-by: Brian Behlendorf Signed-off-by: Weigang Li Signed-off-by: Tom Caputi Closes #7305 --- diff --git a/module/zfs/qat_compress.c b/module/zfs/qat_compress.c index 2116c9790..2f7dad4b4 100644 --- a/module/zfs/qat_compress.c +++ b/module/zfs/qat_compress.c @@ -241,18 +241,24 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len, CpaStatus status = CPA_STATUS_SUCCESS; Cpa32U hdr_sz = 0; Cpa32U compressed_sz; - Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 1; - Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 1; + Cpa32U num_src_buf = (src_len >> PAGE_SHIFT) + 2; + Cpa32U num_dst_buf = (dst_len >> PAGE_SHIFT) + 2; Cpa32U bytes_left; char *data; struct page *in_page, *out_page; struct page **in_pages = NULL; struct page **out_pages = NULL; + Cpa32U page_off = 0; struct completion complete; size_t ret = -1; - Cpa16U page_num = 0; + Cpa32U page_num = 0; Cpa16U i; + /* + * We increment num_src_buf and num_dst_buf by 2 to allow + * us to handle non page-aligned buffer addresses and buffers + * whose sizes are not divisible by PAGE_SIZE. + */ Cpa32U src_buffer_list_mem_size = sizeof (CpaBufferList) + (num_src_buf * sizeof (CpaFlatBuffer)); Cpa32U dst_buffer_list_mem_size = sizeof (CpaBufferList) + @@ -306,11 +312,12 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len, data = src; page_num = 0; while (bytes_left > 0) { + page_off = ((long)data & ~PAGE_MASK); in_page = qat_mem_to_page(data); in_pages[page_num] = in_page; - flat_buf_src->pData = kmap(in_page); + flat_buf_src->pData = kmap(in_page) + page_off; flat_buf_src->dataLenInBytes = - min((long)bytes_left, (long)PAGE_SIZE); + min((long)PAGE_SIZE - page_off, (long)bytes_left); bytes_left -= flat_buf_src->dataLenInBytes; data += flat_buf_src->dataLenInBytes; @@ -325,11 +332,12 @@ qat_compress(qat_compress_dir_t dir, char *src, int src_len, data = dst; page_num = 0; while (bytes_left > 0) { + page_off = ((long)data & ~PAGE_MASK); out_page = qat_mem_to_page(data); - flat_buf_dst->pData = kmap(out_page); + flat_buf_dst->pData = kmap(out_page) + page_off; out_pages[page_num] = out_page; flat_buf_dst->dataLenInBytes = - min((long)bytes_left, (long)PAGE_SIZE); + min((long)PAGE_SIZE - page_off, (long)bytes_left); bytes_left -= flat_buf_dst->dataLenInBytes; data += flat_buf_dst->dataLenInBytes; diff --git a/module/zfs/qat_crypt.c b/module/zfs/qat_crypt.c index 3e2a76ab2..b9931e404 100644 --- a/module/zfs/qat_crypt.c +++ b/module/zfs/qat_crypt.c @@ -266,13 +266,11 @@ qat_init_cy_buffer_lists(CpaInstanceHandle inst_handle, uint32_t nr_bufs, if (status != CPA_STATUS_SUCCESS) return (status); - src->numBuffers = nr_bufs; status = QAT_PHYS_CONTIG_ALLOC(&src->pPrivateMetaData, meta_size); if (status != CPA_STATUS_SUCCESS) goto error; if (src != dst) { - dst->numBuffers = nr_bufs; status = QAT_PHYS_CONTIG_ALLOC(&dst->pPrivateMetaData, meta_size); if (status != CPA_STATUS_SUCCESS) @@ -297,10 +295,9 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, CpaStatus status = CPA_STATUS_SUCCESS; Cpa16U i; CpaInstanceHandle cy_inst_handle; - Cpa16U nr_bufs = (enc_len + PAGE_CACHE_SIZE - 1) / PAGE_CACHE_SIZE; + Cpa16U nr_bufs = (enc_len >> PAGE_SHIFT) + 2; Cpa32U bytes_left = 0; - Cpa8S *in = NULL; - Cpa8S *out = NULL; + Cpa8S *data = NULL; CpaCySymSessionCtx *cy_session_ctx = NULL; cy_callback_t cb; CpaCySymOpData op_data = { 0 }; @@ -312,7 +309,8 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, CpaFlatBuffer *flat_dst_buf = NULL; struct page *in_pages[MAX_PAGE_NUM]; struct page *out_pages[MAX_PAGE_NUM]; - Cpa32S page_num = 0; + Cpa32U in_page_num = 0; + Cpa32U out_page_num = 0; Cpa32U in_page_off = 0; Cpa32U out_page_off = 0; @@ -336,6 +334,11 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, return (status); } + /* + * We increment nr_bufs by 2 to allow us to handle non + * page-aligned buffer addresses and buffers whose sizes + * are not divisible by PAGE_SIZE. + */ status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, &src_buffer_list, &dst_buffer_list); if (status != CPA_STATUS_SUCCESS) @@ -351,30 +354,39 @@ qat_crypt(qat_encrypt_dir_t dir, uint8_t *src_buf, uint8_t *dst_buf, goto fail; bytes_left = enc_len; - in = src_buf; - out = dst_buf; + data = src_buf; flat_src_buf = flat_src_buf_array; - flat_dst_buf = flat_dst_buf_array; while (bytes_left > 0) { - in_page_off = ((long)in & ~PAGE_MASK); - out_page_off = ((long)out & ~PAGE_MASK); - in_pages[page_num] = qat_mem_to_page(in); - out_pages[page_num] = qat_mem_to_page(out); - flat_src_buf->pData = kmap(in_pages[page_num]) + in_page_off; - flat_dst_buf->pData = kmap(out_pages[page_num]) + out_page_off; + in_page_off = ((long)data & ~PAGE_MASK); + in_pages[in_page_num] = qat_mem_to_page(data); + flat_src_buf->pData = kmap(in_pages[in_page_num]) + in_page_off; flat_src_buf->dataLenInBytes = - min((long)PAGE_CACHE_SIZE - in_page_off, (long)bytes_left); - flat_dst_buf->dataLenInBytes = - min((long)PAGE_CACHE_SIZE - out_page_off, (long)bytes_left); - in += flat_src_buf->dataLenInBytes; - out += flat_dst_buf->dataLenInBytes; + min((long)PAGE_SIZE - in_page_off, (long)bytes_left); + data += flat_src_buf->dataLenInBytes; bytes_left -= flat_src_buf->dataLenInBytes; flat_src_buf++; - flat_dst_buf++; - page_num++; + in_page_num++; } src_buffer_list.pBuffers = flat_src_buf_array; + src_buffer_list.numBuffers = in_page_num; + + bytes_left = enc_len; + data = dst_buf; + flat_dst_buf = flat_dst_buf_array; + while (bytes_left > 0) { + out_page_off = ((long)data & ~PAGE_MASK); + out_pages[out_page_num] = qat_mem_to_page(data); + flat_dst_buf->pData = kmap(out_pages[out_page_num]) + + out_page_off; + flat_dst_buf->dataLenInBytes = + min((long)PAGE_SIZE - out_page_off, (long)bytes_left); + data += flat_dst_buf->dataLenInBytes; + bytes_left -= flat_dst_buf->dataLenInBytes; + flat_dst_buf++; + out_page_num++; + } dst_buffer_list.pBuffers = flat_dst_buf_array; + dst_buffer_list.numBuffers = out_page_num; op_data.sessionCtx = cy_session_ctx; op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL; @@ -418,10 +430,10 @@ fail: if (status != CPA_STATUS_SUCCESS) QAT_STAT_BUMP(crypt_fails); - for (i = 0; i < page_num; i++) { + for (i = 0; i < in_page_num; i++) kunmap(in_pages[i]); + for (i = 0; i < out_page_num; i++) kunmap(out_pages[i]); - } cpaCySymRemoveSession(cy_inst_handle, cy_session_ctx); QAT_PHYS_CONTIG_FREE(src_buffer_list.pPrivateMetaData); @@ -439,7 +451,7 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) CpaStatus status; Cpa16U i; CpaInstanceHandle cy_inst_handle; - Cpa16U nr_bufs = (size + PAGE_CACHE_SIZE - 1) / PAGE_CACHE_SIZE; + Cpa16U nr_bufs = (size >> PAGE_SHIFT) + 2; Cpa32U bytes_left = 0; Cpa8S *data = NULL; CpaCySymSessionCtx *cy_session_ctx = NULL; @@ -450,7 +462,7 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) CpaFlatBuffer *flat_src_buf_array = NULL; CpaFlatBuffer *flat_src_buf = NULL; struct page *in_pages[MAX_PAGE_NUM]; - Cpa32S page_num = 0; + Cpa32U page_num = 0; Cpa32U page_off = 0; QAT_STAT_BUMP(cksum_requests); @@ -469,6 +481,11 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) return (status); } + /* + * We increment nr_bufs by 2 to allow us to handle non + * page-aligned buffer addresses and buffers whose sizes + * are not divisible by PAGE_SIZE. + */ status = qat_init_cy_buffer_lists(cy_inst_handle, nr_bufs, &src_buffer_list, &src_buffer_list); if (status != CPA_STATUS_SUCCESS) @@ -487,13 +504,14 @@ qat_checksum(uint64_t cksum, uint8_t *buf, uint64_t size, zio_cksum_t *zcp) in_pages[page_num] = qat_mem_to_page(data); flat_src_buf->pData = kmap(in_pages[page_num]) + page_off; flat_src_buf->dataLenInBytes = - min((long)PAGE_CACHE_SIZE - page_off, (long)bytes_left); + min((long)PAGE_SIZE - page_off, (long)bytes_left); data += flat_src_buf->dataLenInBytes; bytes_left -= flat_src_buf->dataLenInBytes; flat_src_buf++; page_num++; } src_buffer_list.pBuffers = flat_src_buf_array; + src_buffer_list.numBuffers = page_num; op_data.sessionCtx = cy_session_ctx; op_data.packetType = CPA_CY_SYM_PACKET_TYPE_FULL;