"old ones.\n"));
break;
+ case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
+ (void) printf(gettext(" action: Existing "
+ "encrypted datasets contain an on-disk "
+ "incompatibility which\n\tmay cause "
+ "on-disk corruption with 'zfs recv' and "
+ "which needs to be\n\tcorrected. Enable "
+ "the bookmark_v2 feature and backup "
+ "these datasets to new encrypted "
+ "datasets and\n\tdestroy the "
+ "old ones.\n"));
+ break;
default:
/*
* All errata must contain an action message.
"mount existing encrypted datasets readonly.\n"));
break;
+ case ZPOOL_ERRATA_ZOL_8308_ENCRYPTION:
+ (void) printf(gettext("\tExisting encrypted datasets "
+ "contain an on-disk incompatibility\n\twhich "
+ "needs to be corrected.\n"));
+ (void) printf(gettext("action: To correct the issue "
+ "enable the bookmark_v2 feature and "
+ "backup\n\texisting encrypted datasets to new "
+ "encrypted datasets and\n\tdestroy the old "
+ "ones.\n"));
+ break;
+
default:
/*
* All errata which allow the pool to be imported
struct avl_tree *drc_guid_to_ds_map;
nvlist_t *drc_keynvl;
zio_cksum_t drc_cksum;
+ uint64_t drc_fromsnapobj;
uint64_t drc_newsnapobj;
+ uint64_t drc_ivset_guid;
void *drc_owner;
cred_t *drc_cred;
} dmu_recv_cookie_t;
int spa_keystore_lookup_key(spa_t *spa, uint64_t dsobj, void *tag,
dsl_crypto_key_t **dck_out);
-int dsl_crypto_populate_key_nvlist(struct dsl_dataset *ds, nvlist_t **nvl_out);
+int dsl_crypto_populate_key_nvlist(struct dsl_dataset *ds,
+ uint64_t from_ivset_guid, nvlist_t **nvl_out);
int dsl_crypto_recv_raw_key_check(struct dsl_dataset *ds,
nvlist_t *nvl, dmu_tx_t *tx);
void dsl_crypto_recv_raw_key_sync(struct dsl_dataset *ds,
nvlist_t *nvl, dmu_tx_t *tx);
-int dsl_crypto_recv_raw(const char *poolname, uint64_t dsobj,
+int dsl_crypto_recv_raw(const char *poolname, uint64_t dsobj, uint64_t fromobj,
dmu_objset_type_t ostype, nvlist_t *nvl, boolean_t do_key);
int spa_keystore_change_key(const char *dsname, dsl_crypto_params_t *dcp);
*/
#define DS_FIELD_REMAP_DEADLIST "com.delphix:remap_deadlist"
+/*
+ * This field is set to the ivset guid for encrypted snapshots. This is used
+ * for validating raw receives.
+ */
+#define DS_FIELD_IVSET_GUID "com.datto:ivset_guid"
+
/*
* DS_FLAG_CI_DATASET is set if the dataset contains a file system whose
* name lookups should be performed case-insensitively.
ZFS_PROP_KEYSTATUS,
ZFS_PROP_REMAPTXG, /* not exposed to the user */
ZFS_PROP_SPECIAL_SMALL_BLOCKS,
+ ZFS_PROP_IVSET_GUID, /* not exposed to the user */
ZFS_NUM_PROPS
} zfs_prop_t;
ZPOOL_ERRATA_ZOL_2094_SCRUB,
ZPOOL_ERRATA_ZOL_2094_ASYNC_DESTROY,
ZPOOL_ERRATA_ZOL_6845_ENCRYPTION,
+ ZPOOL_ERRATA_ZOL_8308_ENCRYPTION,
} zpool_errata_t;
/*
ZFS_ERR_IOC_ARG_REQUIRED,
ZFS_ERR_IOC_ARG_BADTYPE,
ZFS_ERR_WRONG_PARENT,
+ ZFS_ERR_FROM_IVSET_GUID_MISSING,
+ ZFS_ERR_FROM_IVSET_GUID_MISMATCH,
} zfs_errno_t;
/*
fnvlist_add_boolean(props, zfs_prop_to_name(ZFS_PROP_GUID));
fnvlist_add_boolean(props, zfs_prop_to_name(ZFS_PROP_CREATETXG));
fnvlist_add_boolean(props, zfs_prop_to_name(ZFS_PROP_CREATION));
+ fnvlist_add_boolean(props, zfs_prop_to_name(ZFS_PROP_IVSET_GUID));
if ((err = lzc_get_bookmarks(zhp->zfs_name, props, &bmarks)) != 0)
goto out;
"destination %s space quota exceeded."), name);
(void) zfs_error(hdl, EZFS_NOSPC, errbuf);
break;
+ case ZFS_ERR_FROM_IVSET_GUID_MISSING:
+ zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+ "IV set guid missing. See errata %u at"
+ "http://zfsonlinux.org/msg/ZFS-8000-ER"),
+ ZPOOL_ERRATA_ZOL_8308_ENCRYPTION);
+ (void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
+ break;
+ case ZFS_ERR_FROM_IVSET_GUID_MISMATCH:
+ zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
+ "IV set guid mismatch. See the 'zfs receive' "
+ "man page section\n discussing the limitations "
+ "of raw encrypted send streams."));
+ (void) zfs_error(hdl, EZFS_BADSTREAM, errbuf);
+ break;
case EBUSY:
if (hastoken) {
zfs_error_aux(hdl, dgettext(TEXT_DOMAIN,
* "guid" - globally unique identifier of the snapshot it refers to
* "createtxg" - txg when the snapshot it refers to was created
* "creation" - timestamp when the snapshot it refers to was created
+ * "ivsetguid" - IVset guid for identifying encrypted snapshots
*
* The format of the returned nvlist as follows:
* <short name of bookmark> -> {
.Fl o
options.
.Pp
+The added security provided by raw sends adds some restrictions to the send
+and receive process. ZFS will not allow a mix of raw receives and non-raw
+receives. Specifically, any raw incremental receives that are attempted after
+a non-raw receive will fail. Non-raw receives do not have this restriction and,
+therefore, are always possible. Because of this, it is best practice to always
+use either raw sends for their security benefits or non-raw sends for their
+flexibility when working with encrypted datasets, but not a combination.
+.Pp
+The reason for this restriction stems from the inherent restrictions of the
+AEAD ciphers that ZFS uses to encrypt data. When using ZFS native encryption,
+each block of data is encrypted against a randomly generated number known as
+the "initialization vector" (IV), which is stored in the filesystem metadata.
+This number is required by the encryption algorithms whenever the data is to
+be decrypted. Together, all of the IVs provided for all of the blocks in a
+given snapshot are collectively called an "IV set". When ZFS performs a raw
+send, the IV set is transferred from the source to the destination in the send
+stream. When ZFS performs a non-raw send, the data is decrypted by the source
+system and re-encrypted by the destination system, creating a snapshot with
+effectively the same data, but a different IV set. In order for decryption to
+work after a raw send, ZFS must ensure that the IV set used on both the source
+and destination side match. When an incremental raw receive is performed on
+top of an existing snapshot, ZFS will check to confirm that the "from"
+snapshot on both the source and destination were using the same IV set,
+ensuring the new IV set is consistent.
+.Pp
The name of the snapshot
.Pq and file system, if a full stream is received
that this subcommand creates depends on the argument type and the use of the
{
static const spa_feature_t bookmark_v2_deps[] = {
SPA_FEATURE_EXTENSIBLE_DATASET,
+ SPA_FEATURE_BOOKMARKS,
SPA_FEATURE_NONE
};
zfeature_register(SPA_FEATURE_BOOKMARK_V2,
{
static const spa_feature_t encryption_deps[] = {
SPA_FEATURE_EXTENSIBLE_DATASET,
+ SPA_FEATURE_BOOKMARK_V2,
SPA_FEATURE_NONE
};
zfeature_register(SPA_FEATURE_ENCRYPTION,
PROP_READONLY, ZFS_TYPE_DATASET, "UNIQUE");
zprop_register_hidden(ZFS_PROP_INCONSISTENT, "inconsistent",
PROP_TYPE_NUMBER, PROP_READONLY, ZFS_TYPE_DATASET, "INCONSISTENT");
+ zprop_register_hidden(ZFS_PROP_IVSET_GUID, "ivsetguid",
+ PROP_TYPE_NUMBER, PROP_READONLY,
+ ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK, "IVSETGUID");
zprop_register_hidden(ZFS_PROP_PREV_SNAP, "prevsnap", PROP_TYPE_STRING,
PROP_READONLY, ZFS_TYPE_FILESYSTEM | ZFS_TYPE_VOLUME, "PREVSNAP");
zprop_register_hidden(ZFS_PROP_PBKDF2_SALT, "pbkdf2salt",
dmu_recv_cookie_t *drba_cookie;
cred_t *drba_cred;
dsl_crypto_params_t *drba_dcp;
- uint64_t drba_snapobj;
} dmu_recv_begin_arg_t;
static int
dsl_dataset_t *snap;
uint64_t obj = dsl_dataset_phys(ds)->ds_prev_snap_obj;
- /* Can't perform a raw receive on top of a non-raw receive */
+ /* Can't raw receive on top of an unencrypted dataset */
if (!encrypted && raw)
return (SET_ERROR(EINVAL));
return (SET_ERROR(ENODEV));
if (drba->drba_cookie->drc_force) {
- drba->drba_snapobj = obj;
+ drba->drba_cookie->drc_fromsnapobj = obj;
} else {
/*
* If we are not forcing, there must be no
dsl_dataset_rele(snap, FTAG);
return (SET_ERROR(ETXTBSY));
}
- drba->drba_snapobj = ds->ds_prev->ds_object;
+ drba->drba_cookie->drc_fromsnapobj =
+ ds->ds_prev->ds_object;
}
dsl_dataset_rele(snap, FTAG);
return (SET_ERROR(EINVAL));
}
- drba->drba_snapobj = 0;
+ drba->drba_cookie->drc_fromsnapobj = 0;
}
return (0);
* the raw cmd set. Raw incremental recvs do not use a dcp
* since the encryption parameters are already set in stone.
*/
- if (dcp == NULL && drba->drba_snapobj == 0 &&
+ if (dcp == NULL && drba->drba_cookie->drc_fromsnapobj == 0 &&
drba->drba_origin == NULL) {
ASSERT3P(dcp, ==, NULL);
dcp = &dummy_dcp;
/* create temporary clone */
dsl_dataset_t *snap = NULL;
- if (drba->drba_snapobj != 0) {
+ if (drba->drba_cookie->drc_fromsnapobj != 0) {
VERIFY0(dsl_dataset_hold_obj(dp,
- drba->drba_snapobj, FTAG, &snap));
+ drba->drba_cookie->drc_fromsnapobj, FTAG, &snap));
ASSERT3P(dcp, ==, NULL);
}
dsobj = dsl_dataset_create_sync(ds->ds_dir, recv_clone_name,
snap, crflags, drba->drba_cred, dcp, tx);
- if (drba->drba_snapobj != 0)
+ if (drba->drba_cookie->drc_fromsnapobj != 0)
dsl_dataset_rele(snap, FTAG);
dsl_dataset_rele_flags(ds, dsflags, FTAG);
} else {
* the keynvl away until then.
*/
err = dsl_crypto_recv_raw(spa_name(ra->os->os_spa),
- drc->drc_ds->ds_object, drc->drc_drrb->drr_type,
- keynvl, drc->drc_newfs);
+ drc->drc_ds->ds_object, drc->drc_fromsnapobj,
+ drc->drc_drrb->drr_type, keynvl, drc->drc_newfs);
if (err != 0)
goto out;
+ /* see comment in dmu_recv_end_sync() */
+ drc->drc_ivset_guid = 0;
+ (void) nvlist_lookup_uint64(keynvl, "to_ivset_guid",
+ &drc->drc_ivset_guid);
+
if (!drc->drc_newfs)
drc->drc_keynvl = fnvlist_dup(keynvl);
}
sizeof (struct receive_record_arg) + ra->rrd->payload_size);
ra->rrd = NULL;
}
- if (ra->next_rrd == NULL)
- ra->next_rrd = kmem_zalloc(sizeof (*ra->next_rrd), KM_SLEEP);
- ra->next_rrd->eos_marker = B_TRUE;
- bqueue_enqueue(&rwa->q, ra->next_rrd, 1);
+ ASSERT3P(ra->rrd, ==, NULL);
+ ra->rrd = kmem_zalloc(sizeof (*ra->rrd), KM_SLEEP);
+ ra->rrd->eos_marker = B_TRUE;
+ bqueue_enqueue(&rwa->q, ra->rrd, 1);
mutex_enter(&rwa->mutex);
while (!rwa->done) {
err = rwa->err;
out:
+ /*
+ * If we hit an error before we started the receive_writer_thread
+ * we need to clean up the next_rrd we create by processing the
+ * DRR_BEGIN record.
+ */
+ if (ra->next_rrd != NULL)
+ kmem_free(ra->next_rrd, sizeof (*ra->next_rrd));
+
nvlist_free(begin_nvl);
if ((featureflags & DMU_BACKUP_FEATURE_DEDUP) && (cleanup_fd != -1))
zfs_onexit_fd_rele(cleanup_fd);
drc->drc_newsnapobj =
dsl_dataset_phys(drc->drc_ds)->ds_prev_snap_obj;
}
+
+ /*
+ * If this is a raw receive, the crypt_keydata nvlist will include
+ * a to_ivset_guid for us to set on the new snapshot. This value
+ * will override the value generated by the snapshot code. However,
+ * this value may not be present, because older implementations of
+ * the raw send code did not include this value, and we are still
+ * allowed to receive them if the zfs_disable_ivset_guid_check
+ * tunable is set, in which case we will leave the newly-generated
+ * value.
+ */
+ if (drc->drc_raw && drc->drc_ivset_guid != 0) {
+ dmu_object_zapify(dp->dp_meta_objset, drc->drc_newsnapobj,
+ DMU_OT_DSL_DATASET, tx);
+ VERIFY0(zap_update(dp->dp_meta_objset, drc->drc_newsnapobj,
+ DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1,
+ &drc->drc_ivset_guid, tx));
+ }
+
zvol_create_minors(dp->dp_spa, drc->drc_tofs, B_TRUE);
/*
}
if (featureflags & DMU_BACKUP_FEATURE_RAW) {
+ uint64_t ivset_guid = (ancestor_zb != NULL) ?
+ ancestor_zb->zbm_ivset_guid : 0;
+
ASSERT(os->os_encrypted);
- err = dsl_crypto_populate_key_nvlist(to_ds, &keynvl);
+ err = dsl_crypto_populate_key_nvlist(to_ds,
+ ivset_guid, &keynvl);
if (err != 0) {
fnvlist_free(nvl);
goto out;
}
if (fromsnap != 0) {
- zfs_bookmark_phys_t zb;
+ zfs_bookmark_phys_t zb = { 0 };
boolean_t is_clone;
err = dsl_dataset_hold_obj(dp, fromsnap, FTAG, &fromds);
dsl_pool_rele(dp, FTAG);
return (err);
}
- if (!dsl_dataset_is_before(ds, fromds, 0))
+ if (!dsl_dataset_is_before(ds, fromds, 0)) {
err = SET_ERROR(EXDEV);
+ dsl_dataset_rele(fromds, FTAG);
+ dsl_dataset_rele_flags(ds, dsflags, FTAG);
+ dsl_pool_rele(dp, FTAG);
+ return (err);
+ }
+
zb.zbm_creation_time =
dsl_dataset_phys(fromds)->ds_creation_time;
zb.zbm_creation_txg = dsl_dataset_phys(fromds)->ds_creation_txg;
zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
+
+ if (dsl_dataset_is_zapified(fromds)) {
+ (void) zap_lookup(dp->dp_meta_objset,
+ fromds->ds_object, DS_FIELD_IVSET_GUID, 8, 1,
+ &zb.zbm_ivset_guid);
+ }
+
is_clone = (fromds->ds_dir != ds->ds_dir);
dsl_dataset_rele(fromds, FTAG);
err = dmu_send_impl(FTAG, dp, ds, &zb, is_clone,
}
if (fromsnap != NULL) {
- zfs_bookmark_phys_t zb;
+ zfs_bookmark_phys_t zb = { 0 };
boolean_t is_clone = B_FALSE;
int fsnamelen = strchr(tosnap, '@') - tosnap;
dsl_dataset_phys(fromds)->ds_creation_txg;
zb.zbm_guid = dsl_dataset_phys(fromds)->ds_guid;
is_clone = (ds->ds_dir != fromds->ds_dir);
+
+ if (dsl_dataset_is_zapified(fromds)) {
+ (void) zap_lookup(dp->dp_meta_objset,
+ fromds->ds_object,
+ DS_FIELD_IVSET_GUID, 8, 1,
+ &zb.zbm_ivset_guid);
+ }
dsl_dataset_rele(fromds, FTAG);
}
} else {
bmark_phys.zbm_creation_time =
dsl_dataset_phys(snapds)->ds_creation_time;
+ /*
+ * If the dataset is encrypted create a larger bookmark to
+ * accommodate the IVset guid. The IVset guid was added
+ * after the encryption feature to prevent a problem with
+ * raw sends. If we encounter an encrypted dataset without
+ * an IVset guid we fall back to a normal bookmark.
+ */
+ if (snapds->ds_dir->dd_crypto_obj != 0 &&
+ spa_feature_is_enabled(dp->dp_spa,
+ SPA_FEATURE_BOOKMARK_V2)) {
+ int err = zap_lookup(mos, snapds->ds_object,
+ DS_FIELD_IVSET_GUID, sizeof (uint64_t), 1,
+ &bmark_phys.zbm_ivset_guid);
+ if (err == 0) {
+ bmark_len = BOOKMARK_PHYS_SIZE_V2;
+ spa_feature_incr(dp->dp_spa,
+ SPA_FEATURE_BOOKMARK_V2, tx);
+ }
+ }
+
VERIFY0(zap_add(mos, bmark_fs->ds_bookmarks,
shortname, sizeof (uint64_t),
bmark_len / sizeof (uint64_t), &bmark_phys, tx));
zap_cursor_retrieve(&zc, &attr) == 0;
zap_cursor_advance(&zc)) {
char *bmark_name = attr.za_name;
- zfs_bookmark_phys_t bmark_phys;
+ zfs_bookmark_phys_t bmark_phys = { 0 };
err = dsl_dataset_bmark_lookup(ds, bmark_name, &bmark_phys);
ASSERT3U(err, !=, ENOENT);
dsl_prop_nvlist_add_uint64(out_props,
ZFS_PROP_CREATION, bmark_phys.zbm_creation_time);
}
+ if (nvlist_exists(props,
+ zfs_prop_to_name(ZFS_PROP_IVSET_GUID))) {
+ dsl_prop_nvlist_add_uint64(out_props,
+ ZFS_PROP_IVSET_GUID, bmark_phys.zbm_ivset_guid);
+ }
fnvlist_add_nvlist(outnvl, bmark_name, out_props);
fnvlist_free(out_props);
static int
dsl_dataset_bookmark_remove(dsl_dataset_t *ds, const char *name, dmu_tx_t *tx)
{
+ int err;
objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
uint64_t bmark_zapobj = ds->ds_bookmarks;
matchtype_t mt = 0;
+ uint64_t int_size, num_ints;
if (dsl_dataset_phys(ds)->ds_flags & DS_FLAG_CI_DATASET)
mt = MT_NORMALIZE;
+ err = zap_length(mos, bmark_zapobj, name, &int_size, &num_ints);
+ if (err != 0)
+ return (err);
+
+ ASSERT3U(int_size, ==, sizeof (uint64_t));
+
+ if (num_ints * int_size > BOOKMARK_PHYS_SIZE_V1) {
+ spa_feature_decr(dmu_objset_spa(mos),
+ SPA_FEATURE_BOOKMARK_V2, tx);
+ }
+
return (zap_remove_norm(mos, bmark_zapobj, name, mt, tx));
}
* object is also refcounted.
*/
+/*
+ * This tunable allows datasets to be raw received even if the stream does
+ * not include IVset guids or if the guids don't match. This is used as part
+ * of the resolution for ZPOOL_ERRATA_ZOL_8308_ENCRYPTION.
+ */
+int zfs_disable_ivset_guid_check = 0;
+
static void
dsl_wrapping_key_hold(dsl_wrapping_key_t *wkey, void *tag)
{
typedef struct dsl_crypto_recv_key_arg {
uint64_t dcrka_dsobj;
+ uint64_t dcrka_fromobj;
dmu_objset_type_t dcrka_ostype;
nvlist_t *dcrka_nvl;
boolean_t dcrka_do_key;
} dsl_crypto_recv_key_arg_t;
static int
-dsl_crypto_recv_raw_objset_check(dsl_dataset_t *ds, dmu_objset_type_t ostype,
- nvlist_t *nvl, dmu_tx_t *tx)
+dsl_crypto_recv_raw_objset_check(dsl_dataset_t *ds, dsl_dataset_t *fromds,
+ dmu_objset_type_t ostype, nvlist_t *nvl, dmu_tx_t *tx)
{
int ret;
objset_t *os;
dnode_t *mdn;
uint8_t *buf = NULL;
uint_t len;
- uint64_t intval, nlevels, blksz, ibs, nblkptr, maxblkid;
+ uint64_t intval, nlevels, blksz, ibs;
+ uint64_t nblkptr, maxblkid;
if (ostype != DMU_OST_ZFS && ostype != DMU_OST_ZVOL)
return (SET_ERROR(EINVAL));
}
rrw_exit(&ds->ds_bp_rwlock, FTAG);
+ /*
+ * Check that the ivset guid of the fromds matches the one from the
+ * send stream. Older versions of the encryption code did not have
+ * an ivset guid on the from dataset and did not send one in the
+ * stream. For these streams we provide the
+ * zfs_disable_ivset_guid_check tunable to allow these datasets to
+ * be received with a generated ivset guid.
+ */
+ if (fromds != NULL && !zfs_disable_ivset_guid_check) {
+ uint64_t from_ivset_guid = 0;
+ intval = 0;
+
+ (void) nvlist_lookup_uint64(nvl, "from_ivset_guid", &intval);
+ (void) zap_lookup(tx->tx_pool->dp_meta_objset,
+ fromds->ds_object, DS_FIELD_IVSET_GUID,
+ sizeof (from_ivset_guid), 1, &from_ivset_guid);
+
+ if (intval == 0 || from_ivset_guid == 0)
+ return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISSING));
+
+ if (intval != from_ivset_guid)
+ return (SET_ERROR(ZFS_ERR_FROM_IVSET_GUID_MISMATCH));
+ }
+
return (0);
}
VERIFY0(dmu_objset_from_ds(ds, &os));
mdn = DMU_META_DNODE(os);
- /* fetch the values we need from the nvlist */
+ /*
+ * Fetch the values we need from the nvlist. "to_ivset_guid" must
+ * be set on the snapshot, which doesn't exist yet. The receive
+ * code will take care of this for us later.
+ */
compress = fnvlist_lookup_uint64(nvl, "mdn_compress");
checksum = fnvlist_lookup_uint64(nvl, "mdn_checksum");
nlevels = fnvlist_lookup_uint64(nvl, "mdn_nlevels");
objset_t *mos = tx->tx_pool->dp_meta_objset;
uint8_t *buf = NULL;
uint_t len;
- uint64_t intval, guid, version;
+ uint64_t intval, key_guid, version;
boolean_t is_passphrase = B_FALSE;
ASSERT(dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT);
*/
if (ds->ds_dir->dd_crypto_obj != 0) {
ret = zap_lookup(mos, ds->ds_dir->dd_crypto_obj,
- DSL_CRYPTO_KEY_GUID, 8, 1, &guid);
+ DSL_CRYPTO_KEY_GUID, 8, 1, &key_guid);
if (ret != 0)
return (ret);
- if (intval != guid)
+ if (intval != key_guid)
return (SET_ERROR(EACCES));
}
uint_t len;
uint64_t rddobj, one = 1;
uint8_t *keydata, *hmac_keydata, *iv, *mac;
- uint64_t crypt, guid, keyformat, iters, salt;
+ uint64_t crypt, key_guid, keyformat, iters, salt;
uint64_t version = ZIO_CRYPT_KEY_CURRENT_VERSION;
char *keylocation = "prompt";
/* lookup the values we need to create the DSL Crypto Key */
crypt = fnvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE);
- guid = fnvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_GUID);
+ key_guid = fnvlist_lookup_uint64(nvl, DSL_CRYPTO_KEY_GUID);
keyformat = fnvlist_lookup_uint64(nvl,
zfs_prop_to_name(ZFS_PROP_KEYFORMAT));
iters = fnvlist_lookup_uint64(nvl,
/* sync the key data to the ZAP object on disk */
dsl_crypto_key_sync_impl(mos, dd->dd_crypto_obj, crypt,
- rddobj, guid, iv, mac, keydata, hmac_keydata, keyformat, salt,
+ rddobj, key_guid, iv, mac, keydata, hmac_keydata, keyformat, salt,
iters, tx);
}
{
int ret;
dsl_crypto_recv_key_arg_t *dcrka = arg;
- dsl_dataset_t *ds = NULL;
+ dsl_dataset_t *ds = NULL, *fromds = NULL;
ret = dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_dsobj,
FTAG, &ds);
if (ret != 0)
- goto error;
+ goto out;
- ret = dsl_crypto_recv_raw_objset_check(ds,
+ if (dcrka->dcrka_fromobj != 0) {
+ ret = dsl_dataset_hold_obj(tx->tx_pool, dcrka->dcrka_fromobj,
+ FTAG, &fromds);
+ if (ret != 0)
+ goto out;
+ }
+
+ ret = dsl_crypto_recv_raw_objset_check(ds, fromds,
dcrka->dcrka_ostype, dcrka->dcrka_nvl, tx);
if (ret != 0)
- goto error;
+ goto out;
/*
* We run this check even if we won't be doing this part of
*/
ret = dsl_crypto_recv_raw_key_check(ds, dcrka->dcrka_nvl, tx);
if (ret != 0)
- goto error;
-
- dsl_dataset_rele(ds, FTAG);
- return (0);
+ goto out;
-error:
+out:
if (ds != NULL)
dsl_dataset_rele(ds, FTAG);
+ if (fromds != NULL)
+ dsl_dataset_rele(fromds, FTAG);
return (ret);
}
* without wrapping it.
*/
int
-dsl_crypto_recv_raw(const char *poolname, uint64_t dsobj,
+dsl_crypto_recv_raw(const char *poolname, uint64_t dsobj, uint64_t fromobj,
dmu_objset_type_t ostype, nvlist_t *nvl, boolean_t do_key)
{
dsl_crypto_recv_key_arg_t dcrka;
dcrka.dcrka_dsobj = dsobj;
+ dcrka.dcrka_fromobj = fromobj;
dcrka.dcrka_ostype = ostype;
dcrka.dcrka_nvl = nvl;
dcrka.dcrka_do_key = do_key;
}
int
-dsl_crypto_populate_key_nvlist(dsl_dataset_t *ds, nvlist_t **nvl_out)
+dsl_crypto_populate_key_nvlist(dsl_dataset_t *ds, uint64_t from_ivset_guid,
+ nvlist_t **nvl_out)
{
int ret;
objset_t *os;
dsl_dir_t *rdd = NULL;
dsl_pool_t *dp = ds->ds_dir->dd_pool;
objset_t *mos = dp->dp_meta_objset;
- uint64_t crypt = 0, guid = 0, format = 0;
+ uint64_t crypt = 0, key_guid = 0, format = 0;
uint64_t iters = 0, salt = 0, version = 0;
+ uint64_t to_ivset_guid = 0;
uint8_t raw_keydata[MASTER_KEY_MAX_LEN];
uint8_t raw_hmac_keydata[SHA512_HMAC_KEYLEN];
uint8_t iv[WRAPPING_IV_LEN];
if (ret != 0)
goto error;
- ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, &guid);
+ ret = zap_lookup(mos, dckobj, DSL_CRYPTO_KEY_GUID, 8, 1, &key_guid);
if (ret != 0)
goto error;
if (ret != 0)
goto error;
+ /* see zfs_disable_ivset_guid_check tunable for errata info */
+ ret = zap_lookup(mos, ds->ds_object, DS_FIELD_IVSET_GUID, 8, 1,
+ &to_ivset_guid);
+ if (ret != 0)
+ ASSERT3U(dp->dp_spa->spa_errata, !=, 0);
+
/*
* We don't support raw sends of legacy on-disk formats. See the
* comment in dsl_crypto_recv_key_check() for details.
dsl_pool_config_exit(dp, FTAG);
fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_CRYPTO_SUITE, crypt);
- fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_GUID, guid);
+ fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_GUID, key_guid);
fnvlist_add_uint64(nvl, DSL_CRYPTO_KEY_VERSION, version);
VERIFY0(nvlist_add_uint8_array(nvl, DSL_CRYPTO_KEY_MASTER_KEY,
raw_keydata, MASTER_KEY_MAX_LEN));
fnvlist_add_uint64(nvl, "mdn_indblkshift", mdn->dn_indblkshift);
fnvlist_add_uint64(nvl, "mdn_nblkptr", mdn->dn_nblkptr);
fnvlist_add_uint64(nvl, "mdn_maxblkid", mdn->dn_maxblkid);
+ fnvlist_add_uint64(nvl, "to_ivset_guid", to_ivset_guid);
+ fnvlist_add_uint64(nvl, "from_ivset_guid", from_ivset_guid);
*nvl_out = nvl;
return (0);
zfs_prop_to_name(ZFS_PROP_PBKDF2_ITERS), 8, 1, &intval) == 0) {
dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_PBKDF2_ITERS, intval);
}
+ if (zap_lookup(dd->dd_pool->dp_meta_objset, ds->ds_object,
+ DS_FIELD_IVSET_GUID, 8, 1, &intval) == 0) {
+ dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_IVSET_GUID, intval);
+ }
if (dsl_dir_get_encryption_root_ddobj(dd, &intval) == 0) {
VERIFY0(dsl_dir_hold_obj(dd->dd_pool, intval, NULL, FTAG,
return (ret);
}
+
+#if defined(_KERNEL)
+module_param(zfs_disable_ivset_guid_check, int, 0644);
+MODULE_PARM_DESC(zfs_disable_ivset_guid_check,
+ "Set to allow raw receives without IVset guids");
+#endif
ds->ds_reserved = ds->ds_quota = 0;
}
+ if (err == 0 && ds->ds_dir->dd_crypto_obj != 0 &&
+ ds->ds_is_snapshot &&
+ zap_contains(mos, dsobj, DS_FIELD_IVSET_GUID) != 0) {
+ dp->dp_spa->spa_errata =
+ ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
+ }
+
dsl_deadlist_open(&ds->ds_deadlist,
mos, dsl_dataset_phys(ds)->ds_deadlist_obj);
uint64_t remap_deadlist_obj =
sizeof (remap_deadlist_obj), 1, &remap_deadlist_obj, tx));
}
+ /*
+ * Create a ivset guid for this snapshot if the dataset is
+ * encrypted. This may be overridden by a raw receive. A
+ * previous implementation of this code did not have this
+ * field as part of the on-disk format for ZFS encryption
+ * (see errata #4). As part of the remediation for this
+ * issue, we ask the user to enable the bookmark_v2 feature
+ * which is now a dependency of the encryption feature. We
+ * use this as a heuristic to determine when the user has
+ * elected to correct any datasets created with the old code.
+ * As a result, we only do this step if the bookmark_v2
+ * feature is enabled, which limits the number of states a
+ * given pool / dataset can be in with regards to terms of
+ * correcting the issue.
+ */
+ if (ds->ds_dir->dd_crypto_obj != 0 &&
+ spa_feature_is_enabled(dp->dp_spa, SPA_FEATURE_BOOKMARK_V2)) {
+ uint64_t ivset_guid = unique_create();
+
+ dmu_object_zapify(mos, dsobj, DMU_OT_DSL_DATASET, tx);
+ VERIFY0(zap_add(mos, dsobj, DS_FIELD_IVSET_GUID,
+ sizeof (ivset_guid), 1, &ivset_guid, tx));
+ }
+
ASSERT3U(dsl_dataset_phys(ds)->ds_prev_snap_txg, <, tx->tx_txg);
dsl_dataset_phys(ds)->ds_prev_snap_obj = dsobj;
dsl_dataset_phys(ds)->ds_prev_snap_txg = crtxg;
return (spa_vdev_err(rvd, VDEV_AUX_CORRUPT_DATA, EIO));
}
+ /*
+ * Encryption was added before bookmark_v2, even though bookmark_v2
+ * is now a dependency. If this pool has encryption enabled without
+ * bookmark_v2, trigger an errata message.
+ */
+ if (spa_feature_is_enabled(spa, SPA_FEATURE_ENCRYPTION) &&
+ !spa_feature_is_enabled(spa, SPA_FEATURE_BOOKMARK_V2)) {
+ spa->spa_errata = ZPOOL_ERRATA_ZOL_8308_ENCRYPTION;
+ }
+
return (0);
}
case ZFS_PROP_INCONSISTENT:
numval = dsl_get_inconsistent(ds);
break;
+ case ZFS_PROP_IVSET_GUID:
+ if (dsl_dataset_is_zapified(ds)) {
+ error = zap_lookup(ds->ds_dir->dd_pool->dp_meta_objset,
+ ds->ds_object, DS_FIELD_IVSET_GUID,
+ sizeof (numval), 1, &numval);
+ } else {
+ error = ENOENT;
+ }
+ break;
case ZFS_PROP_RECEIVE_RESUME_TOKEN: {
char *token = get_receive_resume_stats_impl(ds);
'zpool_import_missing_002_pos', 'zpool_import_missing_003_pos',
'zpool_import_rename_001_pos', 'zpool_import_all_001_pos',
'zpool_import_encrypted', 'zpool_import_encrypted_load',
- 'zpool_import_errata3',
+ 'zpool_import_errata3', 'zpool_import_errata4',
'import_cachefile_device_added',
'import_cachefile_device_removed',
'import_cachefile_device_replaced',
'send-c_embedded_blocks', 'send-c_resume', 'send-cpL_varied_recsize',
'send-c_recv_dedup', 'send_encrypted_files', 'send_encrypted_heirarchy',
'send_encrypted_props', 'send_freeobjects', 'send_realloc_dnode_size',
- 'send_holds', 'send_hole_birth', 'send-wDR_encrypted_zvol']
+ 'send_holds', 'send_hole_birth', 'send_mixed_raw',
+ 'send-wDR_encrypted_zvol']
tags = ['functional', 'rsend']
[tests/functional/scrub_mirror]
"feature@project_quota"
"feature@allocation_classes"
"feature@resilver_defer"
+ "feature@bookmark_v2"
)
fi
zpool_import_rename_001_pos.ksh \
zpool_import_encrypted.ksh \
zpool_import_encrypted_load.ksh \
- zpool_import_errata3.ksh
+ zpool_import_errata3.ksh \
+ zpool_import_errata4.ksh
dist_pkgdata_DATA = \
zpool_import.cfg \
pkgdatadir = $(datadir)/@PACKAGE@/zfs-tests/tests/functional/cli_root/zpool_import/blockfiles
dist_pkgdata_DATA = \
unclean_export.dat.bz2 \
- cryptv0.dat.bz2
+ cryptv0.dat.bz2 \
+ missing_ivset.dat.bz2
uncompress_pool
log_must zpool import -d /$TESTPOOL/ $POOL_NAME
-log_must eval "zpool status $POOL_NAME | grep -q Errata"
+log_must eval "zpool status $POOL_NAME | grep -q Errata" # also detects 'Errata #4'
log_must eval "zpool status $POOL_NAME | grep -q ZFS-8000-ER"
log_must eval "echo 'password' | zfs load-key $POOL_NAME/testfs"
log_must eval "echo 'password' | zfs load-key $POOL_NAME/testvol"
log_must zpool export $POOL_NAME
log_must zpool import -d /$TESTPOOL/ $POOL_NAME
-log_mustnot eval "zpool status $POOL_NAME | grep -q Errata"
-log_mustnot eval "zpool status $POOL_NAME | grep -q ZFS-8000-ER"
+log_mustnot eval "zpool status $POOL_NAME | grep -q 'Errata #3'"
log_pass "Errata 3 is properly handled"
--- /dev/null
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+
+#
+# Copyright (c) 2019 Datto, Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# 'zpool import' should import a pool with Errata #4. Users should be
+# able to set the zfs_disable_ivset_guid_check to continue normal
+# operation and the errata should disappear when no more effected
+# datasets remain.
+#
+# STRATEGY:
+# 1. Import a pre-packaged pool with Errata #4 and verify its state
+# 2. Prepare pool to fix existing datasets
+# 3. Use raw sends to fix datasets
+# 4. Ensure fixed datasets match their initial counterparts
+# 5. Destroy the initial datasets and verify the errata is gone
+#
+
+verify_runnable "global"
+
+POOL_NAME=missing_ivset
+POOL_FILE=missing_ivset.dat
+
+function uncompress_pool
+{
+ log_note "Creating pool from $POOL_FILE"
+ log_must bzcat \
+ $STF_SUITE/tests/functional/cli_root/zpool_import/blockfiles/$POOL_FILE.bz2 \
+ > /$TESTPOOL/$POOL_FILE
+ return 0
+}
+
+function cleanup
+{
+ log_must set_tunable32 zfs_disable_ivset_guid_check 0
+ poolexists $POOL_NAME && log_must zpool destroy $POOL_NAME
+ [[ -e /$TESTPOOL/$POOL_FILE ]] && rm /$TESTPOOL/$POOL_FILE
+ return 0
+}
+log_onexit cleanup
+
+log_assert "Verify that Errata 4 is properly handled"
+
+function has_ivset_guid # dataset
+{
+ ds="$1"
+ ivset_guid=$(get_prop ivsetguid $ds)
+
+ if [ "$ivset_guid" == "-" ]; then
+ return 1
+ else
+ return 0
+ fi
+}
+
+# 1. Import a pre-packaged pool with Errata #4 and verify its state
+uncompress_pool
+log_must zpool import -d /$TESTPOOL/ $POOL_NAME
+log_must eval "zpool status $POOL_NAME | grep -q 'Errata #4'"
+log_must eval "zpool status $POOL_NAME | grep -q ZFS-8000-ER"
+bm2_value=$(zpool get -H -o value feature@bookmark_v2 $POOL_NAME)
+if [ "$bm2_value" != "disabled" ]; then
+ log_fail "initial pool's bookmark_v2 feature is not disabled"
+fi
+
+log_mustnot has_ivset_guid $POOL_NAME/testfs@snap1
+log_mustnot has_ivset_guid $POOL_NAME/testfs@snap2
+log_mustnot has_ivset_guid $POOL_NAME/testfs@snap3
+log_mustnot has_ivset_guid $POOL_NAME/testvol@snap1
+log_mustnot has_ivset_guid $POOL_NAME/testvol@snap2
+log_mustnot has_ivset_guid $POOL_NAME/testvol@snap3
+
+# 2. Prepare pool to fix existing datasets
+log_must zpool set feature@bookmark_v2=enabled $POOL_NAME
+log_must set_tunable32 zfs_disable_ivset_guid_check 1
+log_must zfs create $POOL_NAME/fixed
+
+# 3. Use raw sends to fix datasets
+log_must eval "zfs send -w $POOL_NAME/testfs@snap1 | \
+ zfs recv $POOL_NAME/fixed/testfs"
+log_must eval "zfs send -w -i @snap1 $POOL_NAME/testfs@snap2 | \
+ zfs recv $POOL_NAME/fixed/testfs"
+log_must eval \
+ "zfs send -w -i $POOL_NAME/testfs#snap2 $POOL_NAME/testfs@snap3 | \
+ zfs recv $POOL_NAME/fixed/testfs"
+
+log_must eval "zfs send -w $POOL_NAME/testvol@snap1 | \
+ zfs recv $POOL_NAME/fixed/testvol"
+log_must eval "zfs send -w -i @snap1 $POOL_NAME/testvol@snap2 | \
+ zfs recv $POOL_NAME/fixed/testvol"
+log_must eval \
+ "zfs send -w -i $POOL_NAME/testvol#snap2 $POOL_NAME/testvol@snap3 | \
+ zfs recv $POOL_NAME/fixed/testvol"
+
+# 4. Ensure fixed datasets match their initial counterparts
+log_must eval "echo 'password' | zfs load-key $POOL_NAME/testfs"
+log_must eval "echo 'password' | zfs load-key $POOL_NAME/testvol"
+log_must eval "echo 'password' | zfs load-key $POOL_NAME/fixed/testfs"
+log_must eval "echo 'password' | zfs load-key $POOL_NAME/fixed/testvol"
+log_must zfs mount $POOL_NAME/testfs
+log_must zfs mount $POOL_NAME/fixed/testfs
+block_device_wait
+
+old_mntpnt=$(get_prop mountpoint $POOL_NAME/testfs)
+new_mntpnt=$(get_prop mountpoint $POOL_NAME/fixed/testfs)
+log_must diff -r "$old_mntpnt" "$new_mntpnt"
+log_must diff /dev/zvol/$POOL_NAME/testvol /dev/zvol/$POOL_NAME/fixed/testvol
+
+log_must has_ivset_guid $POOL_NAME/fixed/testfs@snap1
+log_must has_ivset_guid $POOL_NAME/fixed/testfs@snap2
+log_must has_ivset_guid $POOL_NAME/fixed/testfs@snap3
+log_must has_ivset_guid $POOL_NAME/fixed/testvol@snap1
+log_must has_ivset_guid $POOL_NAME/fixed/testvol@snap2
+log_must has_ivset_guid $POOL_NAME/fixed/testvol@snap3
+
+# 5. Destroy the initial datasets and verify the errata is gone
+log_must zfs destroy -r $POOL_NAME/testfs
+log_must zfs destroy -r $POOL_NAME/testvol
+
+log_must zpool export $POOL_NAME
+log_must zpool import -d /$TESTPOOL/ $POOL_NAME
+log_mustnot eval "zpool status $POOL_NAME | grep -q 'Errata #4'"
+log_mustnot eval "zpool status $POOL_NAME | grep -q ZFS-8000-ER"
+log_pass "Errata 4 is properly handled"
send_realloc_dnode_size.ksh \
send_holds.ksh \
send_hole_birth.ksh \
+ send_mixed_raw.ksh \
send-wDR_encrypted_zvol.ksh
dist_pkgdata_DATA = \
--- /dev/null
+#!/bin/ksh -p
+#
+# CDDL HEADER START
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source. A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# CDDL HEADER END
+#
+#
+# Copyright (c) 2019 Datto, Inc. All rights reserved.
+#
+
+. $STF_SUITE/include/libtest.shlib
+
+#
+# DESCRIPTION:
+# Verify that 'zfs receive' produces an error when mixing
+# raw and non-raw sends in a way that would break IV set
+# consistency.
+#
+# STRATEGY:
+# 1. Create an initial dataset with 3 snapshots.
+# 2. Perform a raw send of the first snapshot to 2 other datasets.
+# 3. Perform a non-raw send of the second snapshot to one of
+# the other datasets. Perform a raw send from this dataset to
+# the last one.
+# 4. Attempt to raw send the final snapshot of the first dataset
+# to the other 2 datasets, which should fail.
+# 5. Repeat steps 1-4, but using bookmarks for incremental sends.
+#
+#
+# A B C notes
+# ------------------------------------------------------------------------------
+# snap1 ---raw---> snap1 --raw--> snap1 # all snaps initialized via raw send
+# snap2 -non-raw-> snap2 --raw--> snap2 # A sends non-raw to B, B sends raw to C
+# snap3 ------------raw---------> snap3 # attempt send to C (should fail)
+#
+
+
+verify_runnable "both"
+
+function cleanup
+{
+ datasetexists $TESTPOOL/$TESTFS3 && \
+ log_must zfs destroy -r $TESTPOOL/$TESTFS3
+ datasetexists $TESTPOOL/$TESTFS2 && \
+ log_must zfs destroy -r $TESTPOOL/$TESTFS2
+ datasetexists $TESTPOOL/$TESTFS1 && \
+ log_must zfs destroy -r $TESTPOOL/$TESTFS1
+}
+log_onexit cleanup
+
+log_assert "Mixing raw and non-raw receives should fail"
+
+typeset passphrase="password"
+
+log_must eval "echo $passphrase | zfs create -o encryption=on" \
+ "-o keyformat=passphrase $TESTPOOL/$TESTFS1"
+
+log_must zfs snapshot $TESTPOOL/$TESTFS1@1
+log_must touch /$TESTPOOL/$TESTFS1/a
+log_must zfs snapshot $TESTPOOL/$TESTFS1@2
+log_must touch /$TESTPOOL/$TESTFS1/b
+log_must zfs snapshot $TESTPOOL/$TESTFS1@3
+
+# Testing with snapshots
+log_must eval "zfs send -w $TESTPOOL/$TESTFS1@1 |" \
+ "zfs receive $TESTPOOL/$TESTFS2"
+log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS2"
+log_must eval "zfs send -w $TESTPOOL/$TESTFS2@1 |" \
+ "zfs receive $TESTPOOL/$TESTFS3"
+log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS3"
+
+log_must eval "zfs send -i $TESTPOOL/$TESTFS1@1 $TESTPOOL/$TESTFS1@2 |" \
+ "zfs receive $TESTPOOL/$TESTFS2"
+log_must eval "zfs send -w -i $TESTPOOL/$TESTFS2@1 $TESTPOOL/$TESTFS2@2 |" \
+ "zfs receive $TESTPOOL/$TESTFS3"
+
+log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS1@2 $TESTPOOL/$TESTFS1@3 |" \
+ "zfs receive $TESTPOOL/$TESTFS2"
+log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS2@2 $TESTPOOL/$TESTFS2@3 |" \
+ "zfs receive $TESTPOOL/$TESTFS3"
+
+log_must zfs destroy -r $TESTPOOL/$TESTFS3
+log_must zfs destroy -r $TESTPOOL/$TESTFS2
+
+# Testing with bookmarks
+log_must zfs bookmark $TESTPOOL/$TESTFS1@1 $TESTPOOL/$TESTFS1#b1
+log_must zfs bookmark $TESTPOOL/$TESTFS1@2 $TESTPOOL/$TESTFS1#b2
+
+log_must eval "zfs send -w $TESTPOOL/$TESTFS1@1 |" \
+ "zfs receive $TESTPOOL/$TESTFS2"
+log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS2"
+
+log_must zfs bookmark $TESTPOOL/$TESTFS2@1 $TESTPOOL/$TESTFS2#b1
+
+log_must eval "zfs send -w $TESTPOOL/$TESTFS2@1 |" \
+ "zfs receive $TESTPOOL/$TESTFS3"
+log_must eval "echo $passphrase | zfs load-key $TESTPOOL/$TESTFS3"
+
+log_must eval "zfs send -i $TESTPOOL/$TESTFS1#b1 $TESTPOOL/$TESTFS1@2 |" \
+ "zfs receive $TESTPOOL/$TESTFS2"
+log_must eval "zfs send -w -i $TESTPOOL/$TESTFS2#b1 $TESTPOOL/$TESTFS2@2 |" \
+ "zfs receive $TESTPOOL/$TESTFS3"
+
+log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS1#b2" \
+ "$TESTPOOL/$TESTFS1@3 | zfs receive $TESTPOOL/$TESTFS2"
+log_mustnot eval "zfs send -w -i $TESTPOOL/$TESTFS2#b2" \
+ "$TESTPOOL/$TESTFS2@3 | zfs receive $TESTPOOL/$TESTFS3"
+
+log_pass "Mixing raw and non-raw receives fail as expected"