4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
29 * To handle fault injection, we keep track of a series of zinject_record_t
30 * structures which describe which logical block(s) should be injected with a
31 * fault. These are kept in a global list. Each record corresponds to a given
32 * spa_t and maintains a special hold on the spa_t so that it cannot be deleted
33 * or exported while the injection record exists.
35 * Device level injection is done using the 'zi_guid' field. If this is set, it
36 * means that the error is destined for a particular device, not a piece of
39 * This is a rather poor data structure and algorithm, but we don't expect more
40 * than a few faults at any one time, so it should be sufficient for our needs.
45 #include <sys/zfs_ioctl.h>
46 #include <sys/vdev_impl.h>
47 #include <sys/dmu_objset.h>
48 #include <sys/fs/zfs.h>
50 uint32_t zio_injection_enabled = 0;
53 * Data describing each zinject handler registered on the system, and
54 * contains the list node linking the handler in the global zinject
57 typedef struct inject_handler {
60 zinject_record_t zi_record;
67 * List of all zinject handlers registered on the system, protected by
68 * the inject_lock defined below.
70 static list_t inject_handlers;
73 * This protects insertion into, and traversal of, the inject handler
74 * list defined above; as well as the inject_delay_count. Any time a
75 * handler is inserted or removed from the list, this lock should be
76 * taken as a RW_WRITER; and any time traversal is done over the list
77 * (without modification to it) this lock should be taken as a RW_READER.
79 static krwlock_t inject_lock;
82 * This holds the number of zinject delay handlers that have been
83 * registered on the system. It is protected by the inject_lock defined
84 * above. Thus modifications to this count must be a RW_WRITER of the
85 * inject_lock, and reads of this count must be (at least) a RW_READER
88 static int inject_delay_count = 0;
91 * This lock is used only in zio_handle_io_delay(), refer to the comment
92 * in that function for more details.
94 static kmutex_t inject_delay_mtx;
97 * Used to assign unique identifying numbers to each new zinject handler.
99 static int inject_next_id = 1;
102 * Returns true if the given record matches the I/O in progress.
105 zio_match_handler(zbookmark_phys_t *zb, uint64_t type,
106 zinject_record_t *record, int error)
109 * Check for a match against the MOS, which is based on type
111 if (zb->zb_objset == DMU_META_OBJSET &&
112 record->zi_objset == DMU_META_OBJSET &&
113 record->zi_object == DMU_META_DNODE_OBJECT) {
114 if (record->zi_type == DMU_OT_NONE ||
115 type == record->zi_type)
116 return (record->zi_freq == 0 ||
117 spa_get_random(100) < record->zi_freq);
123 * Check for an exact match.
125 if (zb->zb_objset == record->zi_objset &&
126 zb->zb_object == record->zi_object &&
127 zb->zb_level == record->zi_level &&
128 zb->zb_blkid >= record->zi_start &&
129 zb->zb_blkid <= record->zi_end &&
130 error == record->zi_error)
131 return (record->zi_freq == 0 ||
132 spa_get_random(100) < record->zi_freq);
138 * Panic the system when a config change happens in the function
142 zio_handle_panic_injection(spa_t *spa, char *tag, uint64_t type)
144 inject_handler_t *handler;
146 rw_enter(&inject_lock, RW_READER);
148 for (handler = list_head(&inject_handlers); handler != NULL;
149 handler = list_next(&inject_handlers, handler)) {
151 if (spa != handler->zi_spa)
154 if (handler->zi_record.zi_type == type &&
155 strcmp(tag, handler->zi_record.zi_func) == 0)
156 panic("Panic requested in function %s\n", tag);
159 rw_exit(&inject_lock);
163 * Determine if the I/O in question should return failure. Returns the errno
164 * to be returned to the caller.
167 zio_handle_fault_injection(zio_t *zio, int error)
170 inject_handler_t *handler;
173 * Ignore I/O not associated with any logical data.
175 if (zio->io_logical == NULL)
179 * Currently, we only support fault injection on reads.
181 if (zio->io_type != ZIO_TYPE_READ)
184 rw_enter(&inject_lock, RW_READER);
186 for (handler = list_head(&inject_handlers); handler != NULL;
187 handler = list_next(&inject_handlers, handler)) {
189 if (zio->io_spa != handler->zi_spa ||
190 handler->zi_record.zi_cmd != ZINJECT_DATA_FAULT)
193 /* If this handler matches, return EIO */
194 if (zio_match_handler(&zio->io_logical->io_bookmark,
195 zio->io_bp ? BP_GET_TYPE(zio->io_bp) : DMU_OT_NONE,
196 &handler->zi_record, error)) {
202 rw_exit(&inject_lock);
208 * Determine if the zio is part of a label update and has an injection
209 * handler associated with that portion of the label. Currently, we
210 * allow error injection in either the nvlist or the uberblock region of
214 zio_handle_label_injection(zio_t *zio, int error)
216 inject_handler_t *handler;
217 vdev_t *vd = zio->io_vd;
218 uint64_t offset = zio->io_offset;
222 if (offset >= VDEV_LABEL_START_SIZE &&
223 offset < vd->vdev_psize - VDEV_LABEL_END_SIZE)
226 rw_enter(&inject_lock, RW_READER);
228 for (handler = list_head(&inject_handlers); handler != NULL;
229 handler = list_next(&inject_handlers, handler)) {
230 uint64_t start = handler->zi_record.zi_start;
231 uint64_t end = handler->zi_record.zi_end;
233 if (handler->zi_record.zi_cmd != ZINJECT_LABEL_FAULT)
237 * The injection region is the relative offsets within a
238 * vdev label. We must determine the label which is being
239 * updated and adjust our region accordingly.
241 label = vdev_label_number(vd->vdev_psize, offset);
242 start = vdev_label_offset(vd->vdev_psize, label, start);
243 end = vdev_label_offset(vd->vdev_psize, label, end);
245 if (zio->io_vd->vdev_guid == handler->zi_record.zi_guid &&
246 (offset >= start && offset <= end)) {
251 rw_exit(&inject_lock);
257 zio_handle_device_injection(vdev_t *vd, zio_t *zio, int error)
259 inject_handler_t *handler;
263 * We skip over faults in the labels unless it's during
264 * device open (i.e. zio == NULL).
267 uint64_t offset = zio->io_offset;
269 if (offset < VDEV_LABEL_START_SIZE ||
270 offset >= vd->vdev_psize - VDEV_LABEL_END_SIZE)
274 rw_enter(&inject_lock, RW_READER);
276 for (handler = list_head(&inject_handlers); handler != NULL;
277 handler = list_next(&inject_handlers, handler)) {
279 if (handler->zi_record.zi_cmd != ZINJECT_DEVICE_FAULT)
282 if (vd->vdev_guid == handler->zi_record.zi_guid) {
283 if (handler->zi_record.zi_failfast &&
284 (zio == NULL || (zio->io_flags &
285 (ZIO_FLAG_IO_RETRY | ZIO_FLAG_TRYHARD)))) {
289 /* Handle type specific I/O failures */
291 handler->zi_record.zi_iotype != ZIO_TYPES &&
292 handler->zi_record.zi_iotype != zio->io_type)
295 if (handler->zi_record.zi_error == error) {
297 * For a failed open, pretend like the device
301 vd->vdev_stat.vs_aux =
302 VDEV_AUX_OPEN_FAILED;
305 * Treat these errors as if they had been
306 * retried so that all the appropriate stats
307 * and FMA events are generated.
309 if (!handler->zi_record.zi_failfast &&
311 zio->io_flags |= ZIO_FLAG_IO_RETRY;
316 if (handler->zi_record.zi_error == ENXIO) {
317 ret = SET_ERROR(EIO);
323 rw_exit(&inject_lock);
329 * Simulate hardware that ignores cache flushes. For requested number
330 * of seconds nix the actual writing to disk.
333 zio_handle_ignored_writes(zio_t *zio)
335 inject_handler_t *handler;
337 rw_enter(&inject_lock, RW_READER);
339 for (handler = list_head(&inject_handlers); handler != NULL;
340 handler = list_next(&inject_handlers, handler)) {
342 /* Ignore errors not destined for this pool */
343 if (zio->io_spa != handler->zi_spa ||
344 handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
348 * Positive duration implies # of seconds, negative
351 if (handler->zi_record.zi_timer == 0) {
352 if (handler->zi_record.zi_duration > 0)
353 handler->zi_record.zi_timer = ddi_get_lbolt64();
355 handler->zi_record.zi_timer = zio->io_txg;
358 /* Have a "problem" writing 60% of the time */
359 if (spa_get_random(100) < 60)
360 zio->io_pipeline &= ~ZIO_VDEV_IO_STAGES;
364 rw_exit(&inject_lock);
368 spa_handle_ignored_writes(spa_t *spa)
370 inject_handler_t *handler;
372 if (zio_injection_enabled == 0)
375 rw_enter(&inject_lock, RW_READER);
377 for (handler = list_head(&inject_handlers); handler != NULL;
378 handler = list_next(&inject_handlers, handler)) {
380 if (spa != handler->zi_spa ||
381 handler->zi_record.zi_cmd != ZINJECT_IGNORED_WRITES)
384 if (handler->zi_record.zi_duration > 0) {
385 VERIFY(handler->zi_record.zi_timer == 0 ||
387 (int64_t)handler->zi_record.zi_timer +
388 handler->zi_record.zi_duration * hz,
391 /* duration is negative so the subtraction here adds */
392 VERIFY(handler->zi_record.zi_timer == 0 ||
393 handler->zi_record.zi_timer -
394 handler->zi_record.zi_duration >=
395 spa_syncing_txg(spa));
399 rw_exit(&inject_lock);
403 zio_handle_io_delay(zio_t *zio)
405 vdev_t *vd = zio->io_vd;
406 inject_handler_t *min_handler = NULL;
407 hrtime_t min_target = 0;
408 inject_handler_t *handler;
413 rw_enter(&inject_lock, RW_READER);
416 * inject_delay_count is a subset of zio_injection_enabled that
417 * is only incremented for delay handlers. These checks are
418 * mainly added to remind the reader why we're not explicitly
419 * checking zio_injection_enabled like the other functions.
421 IMPLY(inject_delay_count > 0, zio_injection_enabled > 0);
422 IMPLY(zio_injection_enabled == 0, inject_delay_count == 0);
425 * If there aren't any inject delay handlers registered, then we
426 * can short circuit and simply return 0 here. A value of zero
427 * informs zio_delay_interrupt() that this request should not be
428 * delayed. This short circuit keeps us from acquiring the
429 * inject_delay_mutex unnecessarily.
431 if (inject_delay_count == 0) {
432 rw_exit(&inject_lock);
437 * Each inject handler has a number of "lanes" associated with
438 * it. Each lane is able to handle requests independently of one
439 * another, and at a latency defined by the inject handler
440 * record's zi_timer field. Thus if a handler in configured with
441 * a single lane with a 10ms latency, it will delay requests
442 * such that only a single request is completed every 10ms. So,
443 * if more than one request is attempted per each 10ms interval,
444 * the average latency of the requests will be greater than
445 * 10ms; but if only a single request is submitted each 10ms
446 * interval the average latency will be 10ms.
448 * We need to acquire this mutex to prevent multiple concurrent
449 * threads being assigned to the same lane of a given inject
450 * handler. The mutex allows us to perform the following two
451 * operations atomically:
453 * 1. determine the minimum handler and minimum target
454 * value of all the possible handlers
455 * 2. update that minimum handler's lane array
457 * Without atomicity, two (or more) threads could pick the same
458 * lane in step (1), and then conflict with each other in step
459 * (2). This could allow a single lane handler to process
460 * multiple requests simultaneously, which shouldn't be possible.
462 mutex_enter(&inject_delay_mtx);
464 for (handler = list_head(&inject_handlers);
465 handler != NULL; handler = list_next(&inject_handlers, handler)) {
466 if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)
469 if (handler->zi_record.zi_freq != 0 &&
470 spa_get_random(100) >= handler->zi_record.zi_freq) {
474 if (vd->vdev_guid != handler->zi_record.zi_guid)
478 * Defensive; should never happen as the array allocation
479 * occurs prior to inserting this handler on the list.
481 ASSERT3P(handler->zi_lanes, !=, NULL);
484 * This should never happen, the zinject command should
485 * prevent a user from setting an IO delay with zero lanes.
487 ASSERT3U(handler->zi_record.zi_nlanes, !=, 0);
489 ASSERT3U(handler->zi_record.zi_nlanes, >,
490 handler->zi_next_lane);
493 * We want to issue this IO to the lane that will become
494 * idle the soonest, so we compare the soonest this
495 * specific handler can complete the IO with all other
496 * handlers, to find the lowest value of all possible
497 * lanes. We then use this lane to submit the request.
499 * Since each handler has a constant value for its
500 * delay, we can just use the "next" lane for that
501 * handler; as it will always be the lane with the
502 * lowest value for that particular handler (i.e. the
503 * lane that will become idle the soonest). This saves a
504 * scan of each handler's lanes array.
506 * There's two cases to consider when determining when
507 * this specific IO request should complete. If this
508 * lane is idle, we want to "submit" the request now so
509 * it will complete after zi_timer milliseconds. Thus,
510 * we set the target to now + zi_timer.
512 * If the lane is busy, we want this request to complete
513 * zi_timer milliseconds after the lane becomes idle.
514 * Since the 'zi_lanes' array holds the time at which
515 * each lane will become idle, we use that value to
516 * determine when this request should complete.
518 idle = handler->zi_record.zi_timer + gethrtime();
519 busy = handler->zi_record.zi_timer +
520 handler->zi_lanes[handler->zi_next_lane];
521 target = MAX(idle, busy);
523 if (min_handler == NULL) {
524 min_handler = handler;
529 ASSERT3P(min_handler, !=, NULL);
530 ASSERT3U(min_target, !=, 0);
533 * We don't yet increment the "next lane" variable since
534 * we still might find a lower value lane in another
535 * handler during any remaining iterations. Once we're
536 * sure we've selected the absolute minimum, we'll claim
537 * the lane and increment the handler's "next lane"
541 if (target < min_target) {
542 min_handler = handler;
548 * 'min_handler' will be NULL if no IO delays are registered for
549 * this vdev, otherwise it will point to the handler containing
550 * the lane that will become idle the soonest.
552 if (min_handler != NULL) {
553 ASSERT3U(min_target, !=, 0);
554 min_handler->zi_lanes[min_handler->zi_next_lane] = min_target;
557 * If we've used all possible lanes for this handler,
558 * loop back and start using the first lane again;
559 * otherwise, just increment the lane index.
561 min_handler->zi_next_lane = (min_handler->zi_next_lane + 1) %
562 min_handler->zi_record.zi_nlanes;
565 mutex_exit(&inject_delay_mtx);
566 rw_exit(&inject_lock);
572 * Create a new handler for the given record. We add it to the list, adding
573 * a reference to the spa_t in the process. We increment zio_injection_enabled,
574 * which is the switch to trigger all fault injection.
577 zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record)
579 inject_handler_t *handler;
584 * If this is pool-wide metadata, make sure we unload the corresponding
585 * spa_t, so that the next attempt to load it will trigger the fault.
586 * We call spa_reset() to unload the pool appropriately.
588 if (flags & ZINJECT_UNLOAD_SPA)
589 if ((error = spa_reset(name)) != 0)
592 if (record->zi_cmd == ZINJECT_DELAY_IO) {
594 * A value of zero for the number of lanes or for the
595 * delay time doesn't make sense.
597 if (record->zi_timer == 0 || record->zi_nlanes == 0)
598 return (SET_ERROR(EINVAL));
601 * The number of lanes is directly mapped to the size of
602 * an array used by the handler. Thus, to ensure the
603 * user doesn't trigger an allocation that's "too large"
604 * we cap the number of lanes here.
606 if (record->zi_nlanes >= UINT16_MAX)
607 return (SET_ERROR(EINVAL));
610 if (!(flags & ZINJECT_NULL)) {
612 * spa_inject_ref() will add an injection reference, which will
613 * prevent the pool from being removed from the namespace while
614 * still allowing it to be unloaded.
616 if ((spa = spa_inject_addref(name)) == NULL)
617 return (SET_ERROR(ENOENT));
619 handler = kmem_alloc(sizeof (inject_handler_t), KM_SLEEP);
621 handler->zi_spa = spa;
622 handler->zi_record = *record;
624 if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
625 handler->zi_lanes = kmem_zalloc(
626 sizeof (*handler->zi_lanes) *
627 handler->zi_record.zi_nlanes, KM_SLEEP);
628 handler->zi_next_lane = 0;
630 handler->zi_lanes = NULL;
631 handler->zi_next_lane = 0;
634 rw_enter(&inject_lock, RW_WRITER);
637 * We can't move this increment into the conditional
638 * above because we need to hold the RW_WRITER lock of
639 * inject_lock, and we don't want to hold that while
640 * allocating the handler's zi_lanes array.
642 if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
643 ASSERT3S(inject_delay_count, >=, 0);
644 inject_delay_count++;
645 ASSERT3S(inject_delay_count, >, 0);
648 *id = handler->zi_id = inject_next_id++;
649 list_insert_tail(&inject_handlers, handler);
650 atomic_inc_32(&zio_injection_enabled);
652 rw_exit(&inject_lock);
656 * Flush the ARC, so that any attempts to read this data will end up
657 * going to the ZIO layer. Note that this is a little overkill, but
658 * we don't have the necessary ARC interfaces to do anything else, and
659 * fault injection isn't a performance critical path.
661 if (flags & ZINJECT_FLUSH_ARC)
663 * We must use FALSE to ensure arc_flush returns, since
664 * we're not preventing concurrent ARC insertions.
666 arc_flush(NULL, FALSE);
672 * Returns the next record with an ID greater than that supplied to the
673 * function. Used to iterate over all handlers in the system.
676 zio_inject_list_next(int *id, char *name, size_t buflen,
677 zinject_record_t *record)
679 inject_handler_t *handler;
682 mutex_enter(&spa_namespace_lock);
683 rw_enter(&inject_lock, RW_READER);
685 for (handler = list_head(&inject_handlers); handler != NULL;
686 handler = list_next(&inject_handlers, handler))
687 if (handler->zi_id > *id)
691 *record = handler->zi_record;
692 *id = handler->zi_id;
693 (void) strncpy(name, spa_name(handler->zi_spa), buflen);
696 ret = SET_ERROR(ENOENT);
699 rw_exit(&inject_lock);
700 mutex_exit(&spa_namespace_lock);
706 * Clear the fault handler with the given identifier, or return ENOENT if none
710 zio_clear_fault(int id)
712 inject_handler_t *handler;
714 rw_enter(&inject_lock, RW_WRITER);
716 for (handler = list_head(&inject_handlers); handler != NULL;
717 handler = list_next(&inject_handlers, handler))
718 if (handler->zi_id == id)
721 if (handler == NULL) {
722 rw_exit(&inject_lock);
723 return (SET_ERROR(ENOENT));
726 if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
727 ASSERT3S(inject_delay_count, >, 0);
728 inject_delay_count--;
729 ASSERT3S(inject_delay_count, >=, 0);
732 list_remove(&inject_handlers, handler);
733 rw_exit(&inject_lock);
735 if (handler->zi_record.zi_cmd == ZINJECT_DELAY_IO) {
736 ASSERT3P(handler->zi_lanes, !=, NULL);
737 kmem_free(handler->zi_lanes, sizeof (*handler->zi_lanes) *
738 handler->zi_record.zi_nlanes);
740 ASSERT3P(handler->zi_lanes, ==, NULL);
743 spa_inject_delref(handler->zi_spa);
744 kmem_free(handler, sizeof (inject_handler_t));
745 atomic_dec_32(&zio_injection_enabled);
751 zio_inject_init(void)
753 rw_init(&inject_lock, NULL, RW_DEFAULT, NULL);
754 mutex_init(&inject_delay_mtx, NULL, MUTEX_DEFAULT, NULL);
755 list_create(&inject_handlers, sizeof (inject_handler_t),
756 offsetof(inject_handler_t, zi_link));
760 zio_inject_fini(void)
762 list_destroy(&inject_handlers);
763 mutex_destroy(&inject_delay_mtx);
764 rw_destroy(&inject_lock);
767 #if defined(_KERNEL) && defined(HAVE_SPL)
768 EXPORT_SYMBOL(zio_injection_enabled);
769 EXPORT_SYMBOL(zio_inject_fault);
770 EXPORT_SYMBOL(zio_inject_list_next);
771 EXPORT_SYMBOL(zio_clear_fault);
772 EXPORT_SYMBOL(zio_handle_fault_injection);
773 EXPORT_SYMBOL(zio_handle_device_injection);
774 EXPORT_SYMBOL(zio_handle_label_injection);