Illumos 4897 - Space accounting mismatch in L2ARC/zpool

4897 Space accounting mismatch in L2ARC/zpool

Reviewed by: Matthew Ahrens <mahrens@delphix.com>
Reviewed by: Boris Protopopov <bprotopopov@hotmail.com>
Approved by: Dan McDonald <danmcd@omniti.com>

From the illumos issue tracker:

	L2ARC vdev space usage statistics are calculated as the delta
	between the maximum and minimum vdev offset ever written to
	by the L2ARC fill thread, but do not inform the user of how
	much space in between these two offsets is actually taken up by
	cached buffers. This fix changes that so that vdev space usage
	stats on L2ARC devices accurately track the volume of buffers
	stored on them, allowing users to see the exact L2ARC usage in
	"zpool iostat -v".

References:
  https://www.illumos.org/issues/4897
  https://github.com/illumos/illumos-gate/commit/3038a2b

Ported by: Tim Chase <tim@chase2k.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #2555

diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index ec006cb0f6f3b51dba4f1b9c79d0b5bf2ecc4ea7..ad29237b971ada15e66cd2ffde80a57aa61de4a6 100644 (file)
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -22,7 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
- * Copyright 2013 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
  */
 
 /*
@@ -1701,6 +1701,8 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
                        list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
                        ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
                        ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
+                       vdev_space_update(l2hdr->b_dev->l2ad_vdev,
+                           -l2hdr->b_asize, 0, 0);
                        kmem_cache_free(l2arc_hdr_cache, l2hdr);
                        arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
                        if (hdr->b_state == arc_l2c_only)
@@ -3791,6 +3793,8 @@ arc_release(arc_buf_t *buf, void *tag)
 
        if (l2hdr) {
                ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
+               vdev_space_update(l2hdr->b_dev->l2ad_vdev,
+                   -l2hdr->b_asize, 0, 0);
                kmem_cache_free(l2arc_hdr_cache, l2hdr);
                arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
                ARCSTAT_INCR(arcstat_l2_size, -buf_size);
@@ -4637,6 +4641,7 @@ l2arc_write_done(zio_t *zio)
        arc_buf_hdr_t *head, *ab, *ab_prev;
        l2arc_buf_hdr_t *abl2;
        kmutex_t *hash_lock;
+       int64_t bytes_dropped = 0;
 
        cb = zio->io_private;
        ASSERT(cb != NULL);
@@ -4684,6 +4689,7 @@ l2arc_write_done(zio_t *zio)
                         */
                        list_remove(buflist, ab);
                        ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
+                       bytes_dropped += abl2->b_asize;
                        ab->b_l2hdr = NULL;
                        kmem_cache_free(l2arc_hdr_cache, abl2);
                        arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
@@ -4703,6 +4709,8 @@ l2arc_write_done(zio_t *zio)
        kmem_cache_free(hdr_cache, head);
        mutex_exit(&l2arc_buflist_mtx);
 
+       vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
+
        l2arc_do_free_on_write();
 
        kmem_free(cb, sizeof (l2arc_write_callback_t));
@@ -4841,6 +4849,7 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
        arc_buf_hdr_t *ab, *ab_prev;
        kmutex_t *hash_lock;
        uint64_t taddr;
+       int64_t bytes_evicted = 0;
 
        buflist = dev->l2ad_buflist;
 
@@ -4939,6 +4948,7 @@ top:
                        if (ab->b_l2hdr != NULL) {
                                abl2 = ab->b_l2hdr;
                                ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
+                               bytes_evicted += abl2->b_asize;
                                ab->b_l2hdr = NULL;
                                kmem_cache_free(l2arc_hdr_cache, abl2);
                                arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
@@ -4956,7 +4966,7 @@ top:
        }
        mutex_exit(&l2arc_buflist_mtx);
 
-       vdev_space_update(dev->l2ad_vdev, -(taddr - dev->l2ad_evict), 0, 0);
+       vdev_space_update(dev->l2ad_vdev, -bytes_evicted, 0, 0);
        dev->l2ad_evict = taddr;
 }
 
@@ -5204,15 +5214,13 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
        ARCSTAT_INCR(arcstat_l2_write_bytes, write_asize);
        ARCSTAT_INCR(arcstat_l2_size, write_sz);
        ARCSTAT_INCR(arcstat_l2_asize, write_asize);
-       vdev_space_update(dev->l2ad_vdev, write_psize, 0, 0);
+       vdev_space_update(dev->l2ad_vdev, write_asize, 0, 0);
 
        /*
         * Bump device hand to the device start if it is approaching the end.
         * l2arc_evict() will already have evicted ahead for this case.
         */
        if (dev->l2ad_hand >= (dev->l2ad_end - target_sz)) {
-               vdev_space_update(dev->l2ad_vdev,
-                   dev->l2ad_end - dev->l2ad_hand, 0, 0);
                dev->l2ad_hand = dev->l2ad_start;
                dev->l2ad_evict = dev->l2ad_start;
                dev->l2ad_first = B_FALSE;