From b1d217338a51b025b802ebf6a759f45dcd8e3b4c Mon Sep 17 00:00:00 2001 From: Tom Caputi Date: Wed, 21 Feb 2018 15:28:52 -0500 Subject: [PATCH] Raw receives must compress metadnode blocks Currently, the DMU relies on ZIO layer compression to free LO dnode blocks that no longer have objects in them. However, raw receives disable all compression, meaning that these blocks can never be freed. In addition to the obvious space concerns, this could also cause incremental raw receives to fail to mount since the MAC of a hole is different from that of a completely zeroed block. This patch corrects this issue by adding a special case in zio_write_compress() which will attempt to compress these blocks to a hole even if ZIO_FLAG_RAW_ENCRYPT is set. This patch also removes the zfs_mdcomp_disable tunable, since tuning it could cause these same issues. Reviewed-by: Brian Behlendorf Signed-off-by: Tom Caputi Closes #7198 --- include/sys/dmu.h | 2 -- man/man5/zfs-module-parameters.5 | 11 ---------- module/zfs/arc.c | 3 +++ module/zfs/dbuf.c | 3 ++- module/zfs/dmu.c | 21 ++++++------------- module/zfs/zio.c | 14 ++++++++++++- .../functional/rsend/send_encrypted_files.ksh | 11 ---------- 7 files changed, 24 insertions(+), 41 deletions(-) diff --git a/include/sys/dmu.h b/include/sys/dmu.h index cf9cbaa64..472a4f2b6 100644 --- a/include/sys/dmu.h +++ b/include/sys/dmu.h @@ -1033,8 +1033,6 @@ int dmu_diff(const char *tosnap_name, const char *fromsnap_name, #define ZFS_CRC64_POLY 0xC96C5795D7870F42ULL /* ECMA-182, reflected form */ extern uint64_t zfs_crc64_table[256]; -extern int zfs_mdcomp_disable; - #ifdef __cplusplus } #endif diff --git a/man/man5/zfs-module-parameters.5 b/man/man5/zfs-module-parameters.5 index 529684742..17335ebed 100644 --- a/man/man5/zfs-module-parameters.5 +++ b/man/man5/zfs-module-parameters.5 @@ -1400,17 +1400,6 @@ regardless of this setting. Default value: \fB1,048,576\fR. .RE -.sp -.ne 2 -.na -\fBzfs_mdcomp_disable\fR (int) -.ad -.RS 12n -Disable meta data compression -.sp -Use \fB1\fR for yes and \fB0\fR for no (default). -.RE - .sp .ne 2 .na diff --git a/module/zfs/arc.c b/module/zfs/arc.c index 7aa221b00..a78a029c2 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -6776,6 +6776,9 @@ arc_write_ready(zio_t *zio) buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED; if (BP_GET_COMPRESS(bp) == ZIO_COMPRESS_OFF) buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; + } else if (BP_IS_HOLE(bp) && ARC_BUF_ENCRYPTED(buf)) { + buf->b_flags &= ~ARC_BUF_FLAG_ENCRYPTED; + buf->b_flags &= ~ARC_BUF_FLAG_COMPRESSED; } /* this must be done after the buffer flags are adjusted */ diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c index 51cb0c982..e885a2756 100644 --- a/module/zfs/dbuf.c +++ b/module/zfs/dbuf.c @@ -3465,7 +3465,8 @@ dbuf_check_crypt(dbuf_dirty_record_t *dr) * Writing raw encrypted data requires the db's arc buffer * to be converted to raw by the caller. */ - ASSERT(arc_is_encrypted(db->db_buf)); + ASSERT(arc_is_encrypted(db->db_buf) || + db->db.db_object == DMU_META_DNODE_OBJECT); } } diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index b7c5bba15..e0114e659 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -2107,8 +2107,6 @@ dmu_object_dirty_raw(objset_t *os, uint64_t object, dmu_tx_t *tx) return (err); } -int zfs_mdcomp_disable = 0; - /* * When the "redundant_metadata" property is set to "most", only indirect * blocks of this level and higher will have an additional ditto block. @@ -2138,16 +2136,12 @@ dmu_write_policy(objset_t *os, dnode_t *dn, int level, int wp, zio_prop_t *zp) * 3. all other level 0 blocks */ if (ismd) { - if (zfs_mdcomp_disable) { - compress = ZIO_COMPRESS_EMPTY; - } else { - /* - * XXX -- we should design a compression algorithm - * that specializes in arrays of bps. - */ - compress = zio_compress_select(os->os_spa, - ZIO_COMPRESS_ON, ZIO_COMPRESS_ON); - } + /* + * XXX -- we should design a compression algorithm + * that specializes in arrays of bps. + */ + compress = zio_compress_select(os->os_spa, + ZIO_COMPRESS_ON, ZIO_COMPRESS_ON); /* * Metadata always gets checksummed. If the data @@ -2523,9 +2517,6 @@ EXPORT_SYMBOL(dmu_buf_hold); EXPORT_SYMBOL(dmu_ot); /* BEGIN CSTYLED */ -module_param(zfs_mdcomp_disable, int, 0644); -MODULE_PARM_DESC(zfs_mdcomp_disable, "Disable meta data compression"); - module_param(zfs_nopwrite_enabled, int, 0644); MODULE_PARM_DESC(zfs_nopwrite_enabled, "Enable NOP writes"); diff --git a/module/zfs/zio.c b/module/zfs/zio.c index 9458f3924..7544cf4e3 100644 --- a/module/zfs/zio.c +++ b/module/zfs/zio.c @@ -1532,9 +1532,21 @@ zio_write_compress(zio_t *zio) *bp = zio->io_bp_orig; zio->io_pipeline = zio->io_orig_pipeline; + } else if ((zio->io_flags & ZIO_FLAG_RAW_ENCRYPT) != 0 && + zp->zp_type == DMU_OT_DNODE) { + /* + * The DMU actually relies on the zio layer's compression + * to free metadnode blocks that have had all contained + * dnodes freed. As a result, even when doing a raw + * receive, we must check whether the block can be compressed + * to a hole. + */ + psize = zio_compress_data(ZIO_COMPRESS_EMPTY, + zio->io_abd, NULL, lsize); + if (psize == 0) + compress = ZIO_COMPRESS_OFF; } else { ASSERT3U(psize, !=, 0); - } /* diff --git a/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh b/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh index 20f3788d5..998c2afc3 100755 --- a/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh +++ b/tests/zfs-tests/tests/functional/rsend/send_encrypted_files.ksh @@ -39,11 +39,6 @@ verify_runnable "both" -function set_metadata_compression_disabled # <0|1> -{ - echo $1 > /sys/module/zfs/parameters/zfs_mdcomp_disable -} - function cleanup { datasetexists $TESTPOOL/$TESTFS2 && \ @@ -73,12 +68,6 @@ log_must mkfile 32M /$TESTPOOL/$TESTFS2/truncated log_must truncate -s 4M /$TESTPOOL/$TESTFS2/truncated sync -log_must set_metadata_compression_disabled 1 -log_must dd if=/dev/urandom of=/$TESTPOOL/$TESTFS2/no_mdcomp \ - count=1 bs=512 seek=10G >/dev/null 2>&1 -sync -log_must set_metadata_compression_disabled 0 - log_must mkdir -p /$TESTPOOL/$TESTFS2/dir for i in {1..1000}; do log_must mkfile 512 /$TESTPOOL/$TESTFS2/dir/file-$i -- 2.40.0