Fixes in persistent L2ARC

In l2arc_add_vdev() first decide whether the device is eligible for
L2ARC rebuild or whole device trim and then add it to the list of cache
devices. Otherwise l2arc_feed_thread() might already start writing on
the device invalidating previous content as l2ad_hand = l2ad_start.
However l2arc_rebuild_vdev() needs the device present in the cache
device list to figure out its l2arc_dev_t. Fix this by moving most of
l2arc_rebuild_vdev() in a new function l2arc_rebuild_dev() which does
not need to search in the cache device list.

In contrast to l2arc_add_vdev() we do not have to worry about
l2arc_feed_thread() invalidating previous content when onlining a
cache device. The device parameters (l2ad*) are not cleared when
offlining the device and writing new buffers will not invalidate
all previous content. In worst case only buffers that have not had
their log block written to the device will be lost.

Retire persist_l2arc_00{4,5,8} tests since they cover code already
covered by the remaining ones. Test persist_l2arc_006 is renamed to
persist_l2arc_004 and persist_l2arc_007 is renamed to persist_l2arc_005.

Fix a typo in persist_l2arc_004, and remove an assertion that is not
always true from l2arc_arcstats_pos. Also update an assertion in
persist_l2arc_005 and explain why in a comment.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: George Amanakis <gamanakis@gmail.com>
Closes #12365

10 files changed, 162 insertions, 491 deletions

diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 593ac863c..81cb4dabc 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -9743,6 +9743,80 @@ l2arc_vdev_get(vdev_t *vd)
 	return (dev);
 }
 
+static void
+l2arc_rebuild_dev(l2arc_dev_t *dev, boolean_t reopen)
+{
+	l2arc_dev_hdr_phys_t *l2dhdr = dev->l2ad_dev_hdr;
+	uint64_t l2dhdr_asize = dev->l2ad_dev_hdr_asize;
+	spa_t *spa = dev->l2ad_spa;
+
+	/*
+	 * The L2ARC has to hold at least the payload of one log block for
+	 * them to be restored (persistent L2ARC). The payload of a log block
+	 * depends on the amount of its log entries. We always write log blocks
+	 * with 1022 entries. How many of them are committed or restored depends
+	 * on the size of the L2ARC device. Thus the maximum payload of
+	 * one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device
+	 * is less than that, we reduce the amount of committed and restored
+	 * log entries per block so as to enable persistence.
+	 */
+	if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
+		dev->l2ad_log_entries = 0;
+	} else {
+		dev->l2ad_log_entries = MIN((dev->l2ad_end -
+		    dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
+		    L2ARC_LOG_BLK_MAX_ENTRIES);
+	}
+
+	/*
+	 * Read the device header, if an error is returned do not rebuild L2ARC.
+	 */
+	if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
+		/*
+		 * If we are onlining a cache device (vdev_reopen) that was
+		 * still present (l2arc_vdev_present()) and rebuild is enabled,
+		 * we should evict all ARC buffers and pointers to log blocks
+		 * and reclaim their space before restoring its contents to
+		 * L2ARC.
+		 */
+		if (reopen) {
+			if (!l2arc_rebuild_enabled) {
+				return;
+			} else {
+				l2arc_evict(dev, 0, B_TRUE);
+				/* start a new log block */
+				dev->l2ad_log_ent_idx = 0;
+				dev->l2ad_log_blk_payload_asize = 0;
+				dev->l2ad_log_blk_payload_start = 0;
+			}
+		}
+		/*
+		 * Just mark the device as pending for a rebuild. We won't
+		 * be starting a rebuild in line here as it would block pool
+		 * import. Instead spa_load_impl will hand that off to an
+		 * async task which will call l2arc_spa_rebuild_start.
+		 */
+		dev->l2ad_rebuild = B_TRUE;
+	} else if (spa_writeable(spa)) {
+		/*
+		 * In this case TRIM the whole device if l2arc_trim_ahead > 0,
+		 * otherwise create a new header. We zero out the memory holding
+		 * the header to reset dh_start_lbps. If we TRIM the whole
+		 * device the new header will be written by
+		 * vdev_trim_l2arc_thread() at the end of the TRIM to update the
+		 * trim_state in the header too. When reading the header, if
+		 * trim_state is not VDEV_TRIM_COMPLETE and l2arc_trim_ahead > 0
+		 * we opt to TRIM the whole device again.
+		 */
+		if (l2arc_trim_ahead > 0) {
+			dev->l2ad_trim_all = B_TRUE;
+		} else {
+			bzero(l2dhdr, l2dhdr_asize);
+			l2arc_dev_hdr_update(dev);
+		}
+	}
+}
+
 /*
  * Add a vdev for use by the L2ARC.  By this point the spa has already
  * validated the vdev and opened it.
@@ -9796,98 +9870,51 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd)
 	zfs_refcount_create(&adddev->l2ad_lb_count);
 
 	/*
+	 * Decide if dev is eligible for L2ARC rebuild or whole device
+	 * trimming. This has to happen before the device is added in the
+	 * cache device list and l2arc_dev_mtx is released. Otherwise
+	 * l2arc_feed_thread() might already start writing on the
+	 * device.
+	 */
+	l2arc_rebuild_dev(adddev, B_FALSE);
+
+	/*
 	 * Add device to global list
 	 */
 	mutex_enter(&l2arc_dev_mtx);
 	list_insert_head(l2arc_dev_list, adddev);
 	atomic_inc_64(&l2arc_ndev);
 	mutex_exit(&l2arc_dev_mtx);
-
-	/*
-	 * Decide if vdev is eligible for L2ARC rebuild
-	 */
-	l2arc_rebuild_vdev(adddev->l2ad_vdev, B_FALSE);
 }
 
+/*
+ * Decide if a vdev is eligible for L2ARC rebuild, called from vdev_reopen()
+ * in case of onlining a cache device.
+ */
 void
 l2arc_rebuild_vdev(vdev_t *vd, boolean_t reopen)
 {
 	l2arc_dev_t		*dev = NULL;
-	l2arc_dev_hdr_phys_t	*l2dhdr;
-	uint64_t		l2dhdr_asize;
-	spa_t			*spa;
 
 	dev = l2arc_vdev_get(vd);
 	ASSERT3P(dev, !=, NULL);
-	spa = dev->l2ad_spa;
-	l2dhdr = dev->l2ad_dev_hdr;
-	l2dhdr_asize = dev->l2ad_dev_hdr_asize;
 
 	/*
-	 * The L2ARC has to hold at least the payload of one log block for
-	 * them to be restored (persistent L2ARC). The payload of a log block
-	 * depends on the amount of its log entries. We always write log blocks
-	 * with 1022 entries. How many of them are committed or restored depends
-	 * on the size of the L2ARC device. Thus the maximum payload of
-	 * one log block is 1022 * SPA_MAXBLOCKSIZE = 16GB. If the L2ARC device
-	 * is less than that, we reduce the amount of committed and restored
-	 * log entries per block so as to enable persistence.
-	 */
-	if (dev->l2ad_end < l2arc_rebuild_blocks_min_l2size) {
-		dev->l2ad_log_entries = 0;
-	} else {
-		dev->l2ad_log_entries = MIN((dev->l2ad_end -
-		    dev->l2ad_start) >> SPA_MAXBLOCKSHIFT,
-		    L2ARC_LOG_BLK_MAX_ENTRIES);
-	}
-
-	/*
-	 * Read the device header, if an error is returned do not rebuild L2ARC.
-	 */
-	if (l2arc_dev_hdr_read(dev) == 0 && dev->l2ad_log_entries > 0) {
-		/*
-		 * If we are onlining a cache device (vdev_reopen) that was
-		 * still present (l2arc_vdev_present()) and rebuild is enabled,
-		 * we should evict all ARC buffers and pointers to log blocks
-		 * and reclaim their space before restoring its contents to
-		 * L2ARC.
-		 */
-		if (reopen) {
-			if (!l2arc_rebuild_enabled) {
-				return;
-			} else {
-				l2arc_evict(dev, 0, B_TRUE);
-				/* start a new log block */
-				dev->l2ad_log_ent_idx = 0;
-				dev->l2ad_log_blk_payload_asize = 0;
-				dev->l2ad_log_blk_payload_start = 0;
-			}
-		}
-		/*
-		 * Just mark the device as pending for a rebuild. We won't
-		 * be starting a rebuild in line here as it would block pool
-		 * import. Instead spa_load_impl will hand that off to an
-		 * async task which will call l2arc_spa_rebuild_start.
-		 */
-		dev->l2ad_rebuild = B_TRUE;
-	} else if (spa_writeable(spa)) {
-		/*
-		 * In this case TRIM the whole device if l2arc_trim_ahead > 0,
-		 * otherwise create a new header. We zero out the memory holding
-		 * the header to reset dh_start_lbps. If we TRIM the whole
-		 * device the new header will be written by
-		 * vdev_trim_l2arc_thread() at the end of the TRIM to update the
-		 * trim_state in the header too. When reading the header, if
-		 * trim_state is not VDEV_TRIM_COMPLETE and l2arc_trim_ahead > 0
-		 * we opt to TRIM the whole device again.
-		 */
-		if (l2arc_trim_ahead > 0) {
-			dev->l2ad_trim_all = B_TRUE;
-		} else {
-			bzero(l2dhdr, l2dhdr_asize);
-			l2arc_dev_hdr_update(dev);
-		}
-	}
+	 * In contrast to l2arc_add_vdev() we do not have to worry about
+	 * l2arc_feed_thread() invalidating previous content when onlining a
+	 * cache device. The device parameters (l2ad*) are not cleared when
+	 * offlining the device and writing new buffers will not invalidate
+	 * all previous content. In worst case only buffers that have not had
+	 * their log block written to the device will be lost.
+	 * When onlining the cache device (ie offline->online without exporting
+	 * the pool in between) this happens:
+	 * vdev_reopen() -> vdev_open() -> l2arc_rebuild_vdev()
+	 * 			|			|
+	 * 		vdev_is_dead() = B_FALSE	l2ad_rebuild = B_TRUE
+	 * During the time where vdev_is_dead = B_FALSE and until l2ad_rebuild
+	 * is set to B_TRUE we might write additional buffers to the device.
+	 */
+	l2arc_rebuild_dev(dev, reopen);
 }
 
 /*
diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run
index 5f5e10d13..2421e8aa4 100644
--- a/tests/runfiles/common.run
+++ b/tests/runfiles/common.run
@@ -934,8 +934,7 @@ tags = ['functional', 'log_spacemap']
 [tests/functional/l2arc]
 tests = ['l2arc_arcstats_pos', 'l2arc_mfuonly_pos', 'l2arc_l2miss_pos',
     'persist_l2arc_001_pos', 'persist_l2arc_002_pos',
-    'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos',
-    'persist_l2arc_006_pos', 'persist_l2arc_007_pos', 'persist_l2arc_008_pos']
+    'persist_l2arc_003_neg', 'persist_l2arc_004_pos', 'persist_l2arc_005_pos']
 tags = ['functional', 'l2arc']
 
 [tests/functional/zpool_influxdb]
diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in
index 8c3bce134..4661a47f5 100755
--- a/tests/test-runner/bin/zts-report.py.in
+++ b/tests/test-runner/bin/zts-report.py.in
@@ -223,8 +223,6 @@ maybe = {
     'history/history_008_pos': ['FAIL', known_reason],
     'history/history_010_pos': ['SKIP', exec_reason],
     'io/mmap': ['SKIP', fio_reason],
-    'l2arc/persist_l2arc_005_pos': ['FAIL', known_reason],
-    'l2arc/persist_l2arc_007_pos': ['FAIL', '11887'],
     'largest_pool/largest_pool_001_pos': ['FAIL', known_reason],
     'mmp/mmp_on_uberblocks': ['FAIL', known_reason],
     'pyzfs/pyzfs_unittest': ['SKIP', python_deps_reason],
diff --git a/tests/zfs-tests/tests/functional/l2arc/Makefile.am b/tests/zfs-tests/tests/functional/l2arc/Makefile.am
index 9baf580ee..09f4c1d0d 100644
--- a/tests/zfs-tests/tests/functional/l2arc/Makefile.am
+++ b/tests/zfs-tests/tests/functional/l2arc/Makefile.am
@@ -9,10 +9,7 @@ dist_pkgdata_SCRIPTS = \
 	persist_l2arc_002_pos.ksh \
 	persist_l2arc_003_neg.ksh \
 	persist_l2arc_004_pos.ksh \
-	persist_l2arc_005_pos.ksh \
-	persist_l2arc_006_pos.ksh \
-	persist_l2arc_007_pos.ksh \
-	persist_l2arc_008_pos.ksh
+	persist_l2arc_005_pos.ksh
 
 dist_pkgdata_DATA = \
 	l2arc.cfg
diff --git a/tests/zfs-tests/tests/functional/l2arc/l2arc_arcstats_pos.ksh b/tests/zfs-tests/tests/functional/l2arc/l2arc_arcstats_pos.ksh
index 24fcefadf..3e76347b0 100755
--- a/tests/zfs-tests/tests/functional/l2arc/l2arc_arcstats_pos.ksh
+++ b/tests/zfs-tests/tests/functional/l2arc/l2arc_arcstats_pos.ksh
@@ -96,7 +96,6 @@ typeset l2_mru_end=$(get_arcstat l2_mru_asize)
 typeset l2_prefetch_end=$(get_arcstat l2_prefetch_asize)
 typeset l2_asize_end=$(get_arcstat l2_asize)
 
-log_must test $(( $l2_mfu_end - $l2_mfu_init )) -gt 0
 log_must test $(( $l2_mru_end + $l2_mfu_end + $l2_prefetch_end - \
 	$l2_asize_end )) -eq 0
 log_must test $(( $l2_mru_init + $l2_mfu_init + $l2_prefetch_init - \
diff --git a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_004_pos.ksh b/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_004_pos.ksh
index 544e9291d..b40703180 100755
--- a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_004_pos.ksh
+++ b/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_004_pos.ksh
@@ -23,25 +23,24 @@
 
 #
 # DESCRIPTION:
-#	Persistent L2ARC restores all written log blocks
+#	Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not
+#	present.
 #
 # STRATEGY:
 #	1. Create pool with a cache device.
-#	2. Create a random file in that pool, smaller than the cache device
-#		and random read for 10 sec.
-#	3. Export pool.
-#	4. Read amount of log blocks written.
-#	5. Import pool.
-#	6. Read amount of log blocks built.
-#	7. Compare the two amounts.
-#	8. Read the file written in (2) and check if l2_hits in
-#		/proc/spl/kstat/zfs/arcstats increased.
-#	9. Check if the labels of the L2ARC device are intact.
+#	2. Create a random file in that pool and random read for 10 sec.
+#	3. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	4. Offline the L2ARC device and export pool.
+#	5. Import pool and online the L2ARC device.
+#	6. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(3).
+#	7. Check if the labels of the L2ARC device are intact.
 #
 
 verify_runnable "global"
 
-log_assert "Persistent L2ARC restores all written log blocks."
+log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present."
 
 function cleanup
 {
@@ -50,47 +49,47 @@ function cleanup
 	fi
 
 	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
+		$rebuild_blocks_min_l2size
 }
 log_onexit cleanup
 
 # L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
 typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
 log_must set_tunable32 L2ARC_NOPREFETCH 0
+log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
 
 typeset fill_mb=800
-typeset cache_sz=$(( 2 * $fill_mb ))
+typeset cache_sz=$(( floor($fill_mb / 2) ))
 export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
 
 log_must truncate -s ${cache_sz}M $VDEV_CACHE
 
-typeset log_blk_start=$(get_arcstat l2_log_blk_writes)
-
 log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
 
 log_must fio $FIO_SCRIPTS/mkfiles.fio
 log_must fio $FIO_SCRIPTS/random_reads.fio
 
 arcstat_quiescence_noecho l2_size
+log_must zpool offline $TESTPOOL $VDEV_CACHE
+arcstat_quiescence_noecho l2_size
 log_must zpool export $TESTPOOL
 arcstat_quiescence_noecho l2_feeds
 
-typeset log_blk_end=$(get_arcstat l2_log_blk_writes)
-typeset log_blk_rebuild_start=$(get_arcstat l2_rebuild_log_blks)
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
 
 log_must zpool import -d $VDIR $TESTPOOL
-
-typeset l2_hits_start=$(get_arcstat l2_hits)
-
-log_must fio $FIO_SCRIPTS/random_reads.fio
+log_must zpool online $TESTPOOL $VDEV_CACHE
 arcstat_quiescence_noecho l2_size
 
-typeset log_blk_rebuild_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
-typeset l2_hits_end=$(get_arcstat l2_hits)
-
-log_must test $(( $log_blk_rebuild_end - $log_blk_rebuild_start )) -eq \
-	$(( $log_blk_end - $log_blk_start ))
+typeset l2_rebuild_log_blk_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
 
-log_must test $l2_hits_end -gt $l2_hits_start
+log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - \
+	$l2_rebuild_log_blk_start ))
+log_must test $l2_dh_log_blk -gt 0
 
 log_must zpool offline $TESTPOOL $VDEV_CACHE
 arcstat_quiescence_noecho l2_size
@@ -99,4 +98,4 @@ log_must zdb -lq $VDEV_CACHE
 
 log_must zpool destroy -f $TESTPOOL
 
-log_pass "Persistent L2ARC restores all written log blocks."
+log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present."
diff --git a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_005_pos.ksh b/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_005_pos.ksh
index ee46e7b8c..8ad648519 100755
--- a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_005_pos.ksh
+++ b/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_005_pos.ksh
@@ -20,31 +20,26 @@
 
 . $STF_SUITE/include/libtest.shlib
 . $STF_SUITE/tests/functional/l2arc/l2arc.cfg
-. $STF_SUITE/tests/functional/cli_root/zfs_load-key/zfs_load-key_common.kshlib
 
 #
 # DESCRIPTION:
-#	Persistent L2ARC restores all written log blocks with encryption
+#	Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present.
 #
 # STRATEGY:
 #	1. Create pool with a cache device.
-#	2. Create a an encrypted ZFS file system.
-#	3. Create a random file in the entrypted file system,
-#		smaller than the cache device, and random read for 10 sec.
-#	4. Export pool.
-#	5. Read amount of log blocks written.
-#	6. Import pool.
-#	7. Mount the encrypted ZFS file system.
-#	8. Read amount of log blocks built.
-#	9. Compare the two amounts.
-#	10. Read the file written in (3) and check if l2_hits in
-#		/proc/spl/kstat/zfs/arcstats increased.
-#	11. Check if the labels of the L2ARC device are intact.
+#	2. Create a random file in that pool and random read for 10 sec.
+#	3. Offline the L2ARC device.
+#	4. Read the amount of log blocks written from the header of the
+#		L2ARC device.
+#	5. Online the L2ARC device.
+#	6. Read the amount of log blocks rebuilt in arcstats and compare to
+#		(4).
+#	7. Check if the labels of the L2ARC device are intact.
 #
 
 verify_runnable "global"
 
-log_assert "Persistent L2ARC restores all written log blocks with encryption."
+log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present."
 
 function cleanup
 {
@@ -53,51 +48,49 @@ function cleanup
 	fi
 
 	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
+	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
+		$rebuild_blocks_min_l2size
 }
 log_onexit cleanup
 
 # L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
 typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
+typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
 log_must set_tunable32 L2ARC_NOPREFETCH 0
+log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
 
 typeset fill_mb=800
-typeset cache_sz=$(( 2 * $fill_mb ))
+typeset cache_sz=$(( floor($fill_mb / 2) ))
 export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
 
 log_must truncate -s ${cache_sz}M $VDEV_CACHE
 
-typeset log_blk_start=$(get_arcstat l2_log_blk_writes)
-
 log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
 
-log_must eval "echo $PASSPHRASE | zfs create -o encryption=on" \
-	"-o keyformat=passphrase $TESTPOOL/$TESTFS1"
-
 log_must fio $FIO_SCRIPTS/mkfiles.fio
 log_must fio $FIO_SCRIPTS/random_reads.fio
 
 arcstat_quiescence_noecho l2_size
-log_must zpool export $TESTPOOL
-arcstat_quiescence_noecho l2_feeds
-
-typeset log_blk_end=$(get_arcstat l2_log_blk_writes)
-typeset log_blk_rebuild_start=$(get_arcstat l2_rebuild_log_blks)
-
-log_must zpool import -d $VDIR $TESTPOOL
-log_must eval "echo $PASSPHRASE | zfs mount -l $TESTPOOL/$TESTFS1"
+log_must zpool offline $TESTPOOL $VDEV_CACHE
+arcstat_quiescence_noecho l2_size
 
-typeset l2_hits_start=$(get_arcstat l2_hits)
+typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
+typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
+	awk '{print $2}')
 
-log_must fio $FIO_SCRIPTS/random_reads.fio
+log_must zpool online $TESTPOOL $VDEV_CACHE
 arcstat_quiescence_noecho l2_size
 
-typeset log_blk_rebuild_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
-typeset l2_hits_end=$(get_arcstat l2_hits)
-
-log_must test $(( $log_blk_rebuild_end - $log_blk_rebuild_start )) -eq \
-	$(( $log_blk_end - $log_blk_start ))
+typeset l2_rebuild_log_blk_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
 
-log_must test $l2_hits_end -gt $l2_hits_start
+# Upon onlining the cache device we might write additional blocks to it
+# before it is marked for rebuild as the l2ad_* parameters are not cleared
+# when offlining the device. See comment in l2arc_rebuild_vdev().
+# So we cannot compare the amount of rebuilt log blocks to the amount of log
+# blocks read from the header of the device.
+log_must test $(( $l2_rebuild_log_blk_end - \
+	$l2_rebuild_log_blk_start )) -gt 0
+log_must test $l2_dh_log_blk -gt 0
 
 log_must zpool offline $TESTPOOL $VDEV_CACHE
 arcstat_quiescence_noecho l2_size
@@ -106,4 +99,4 @@ log_must zdb -lq $VDEV_CACHE
 
 log_must zpool destroy -f $TESTPOOL
 
-log_pass "Persistent L2ARC restores all written log blocks with encryption."
+log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present."
diff --git a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_006_pos.ksh b/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_006_pos.ksh
deleted file mode 100755
index 051773540..000000000
--- a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_006_pos.ksh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/ksh -p
-#
-# CDDL HEADER START
-#
-# This file and its contents are supplied under the terms of the
-# Common Development and Distribution License ("CDDL"), version 1.0.
-# You may only use this file in accordance with the terms of version
-# 1.0 of the CDDL.
-#
-# A full copy of the text of the CDDL should have accompanied this
-# source.  A copy of the CDDL is also available via the Internet at
-# http://www.illumos.org/license/CDDL.
-#
-# CDDL HEADER END
-#
-
-#
-# Copyright (c) 2020, George Amanakis. All rights reserved.
-#
-
-. $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/l2arc/l2arc.cfg
-
-#
-# DESCRIPTION:
-#	Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not
-#	present.
-#
-# STRATEGY:
-#	1. Create pool with a cache device.
-#	2. Create a random file in that pool and random read for 10 sec.
-#	3. Read the amount of log blocks written from the header of the
-#		L2ARC device.
-#	4. Offline the L2ARC device and export pool.
-#	5. Import pool and online the L2ARC device.
-#	6. Read the amount of log blocks rebuilt in arcstats and compare to
-#		(3).
-#	7. Check if the labels of the L2ARC device are intact.
-#
-
-verify_runnable "global"
-
-log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present."
-
-function cleanup
-{
-	if poolexists $TESTPOOL ; then
-		destroy_pool $TESTPOOL
-	fi
-
-	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
-	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
-		$rebuild_blocks_min_l2size
-}
-log_onexit cleanup
-
-# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
-typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
-typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
-log_must set_tunable32 L2ARC_NOPREFETCH 0
-log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
-
-typeset fill_mb=800
-typeset cache_sz=$(( floor($fill_mb / 2) ))
-export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
-
-log_must truncate -s ${cache_sz}M $VDEV_CACHE
-
-log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
-
-log_must fio $FIO_SCRIPTS/mkfiles.fio
-log_must fio $FIO_SCRIPTS/random_reads.fio
-
-arcstat_quiescence_noecho l2_size
-log_must zpool offline $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-log_must zpool export $TESTPOOL
-arcstat_quiescence_noecho l2_feeds
-
-typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
-typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
-	awk '{print $2}')
-
-log_must zpool import -d $VDIR $TESTPOOL
-log_must zpool online $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-typeset l2_rebuild_log_blk_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
-
-log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - \
-	$l2_rebuild_log_blk_start ))
-log_must test $l2_dh_log_blk -gt 0
-
-log must zpool offline $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-log_must zdb -lq $VDEV_CACHE
-
-log_must zpool destroy -f $TESTPOOL
-
-log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev not present."
diff --git a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_007_pos.ksh b/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_007_pos.ksh
deleted file mode 100755
index 9208b81d4..000000000
--- a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_007_pos.ksh
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/bin/ksh -p
-#
-# CDDL HEADER START
-#
-# This file and its contents are supplied under the terms of the
-# Common Development and Distribution License ("CDDL"), version 1.0.
-# You may only use this file in accordance with the terms of version
-# 1.0 of the CDDL.
-#
-# A full copy of the text of the CDDL should have accompanied this
-# source.  A copy of the CDDL is also available via the Internet at
-# http://www.illumos.org/license/CDDL.
-#
-# CDDL HEADER END
-#
-
-#
-# Copyright (c) 2020, George Amanakis. All rights reserved.
-#
-
-. $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/l2arc/l2arc.cfg
-
-#
-# DESCRIPTION:
-#	Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present.
-#
-# STRATEGY:
-#	1. Create pool with a cache device.
-#	2. Create a random file in that pool and random read for 10 sec.
-#	3. Offline the L2ARC device.
-#	4. Read the amount of log blocks written from the header of the
-#		L2ARC device.
-#	5. Online the L2ARC device.
-#	6. Read the amount of log blocks rebuilt in arcstats and compare to
-#		(4).
-#	7. Check if the labels of the L2ARC device are intact.
-#
-
-verify_runnable "global"
-
-log_assert "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present."
-
-function cleanup
-{
-	if poolexists $TESTPOOL ; then
-		destroy_pool $TESTPOOL
-	fi
-
-	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
-	log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE \
-		$rebuild_blocks_min_l2size
-}
-log_onexit cleanup
-
-# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
-typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
-typeset rebuild_blocks_min_l2size=$(get_tunable L2ARC_REBUILD_BLOCKS_MIN_L2SIZE)
-log_must set_tunable32 L2ARC_NOPREFETCH 0
-log_must set_tunable32 L2ARC_REBUILD_BLOCKS_MIN_L2SIZE 0
-
-typeset fill_mb=800
-typeset cache_sz=$(( floor($fill_mb / 2) ))
-export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
-
-log_must truncate -s ${cache_sz}M $VDEV_CACHE
-
-log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
-
-log_must fio $FIO_SCRIPTS/mkfiles.fio
-log_must fio $FIO_SCRIPTS/random_reads.fio
-
-arcstat_quiescence_noecho l2_size
-log_must zpool offline $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
-typeset l2_dh_log_blk=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
-	awk '{print $2}')
-
-log_must zpool online $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-typeset l2_rebuild_log_blk_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
-
-log_must test $l2_dh_log_blk -eq $(( $l2_rebuild_log_blk_end - \
-	$l2_rebuild_log_blk_start ))
-log_must test $l2_dh_log_blk -gt 0
-
-log_must zpool offline $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-log_must zdb -lq $VDEV_CACHE
-
-log_must zpool destroy -f $TESTPOOL
-
-log_pass "Off/onlining an L2ARC device results in rebuilding L2ARC, vdev present."
diff --git a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_008_pos.ksh b/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_008_pos.ksh
deleted file mode 100755
index 5a79ff31b..000000000
--- a/tests/zfs-tests/tests/functional/l2arc/persist_l2arc_008_pos.ksh
+++ /dev/null
@@ -1,143 +0,0 @@
-#!/bin/ksh -p
-#
-# CDDL HEADER START
-#
-# This file and its contents are supplied under the terms of the
-# Common Development and Distribution License ("CDDL"), version 1.0.
-# You may only use this file in accordance with the terms of version
-# 1.0 of the CDDL.
-#
-# A full copy of the text of the CDDL should have accompanied this
-# source.  A copy of the CDDL is also available via the Internet at
-# http://www.illumos.org/license/CDDL.
-#
-# CDDL HEADER END
-#
-
-#
-# Copyright (c) 2020, George Amanakis. All rights reserved.
-#
-
-. $STF_SUITE/include/libtest.shlib
-. $STF_SUITE/tests/functional/l2arc/l2arc.cfg
-
-#
-# DESCRIPTION:
-#	Off/onlining an L2ARC device restores all written blocks, vdev present.
-#
-# STRATEGY:
-#	1. Create pool with a cache device.
-#	2. Create a random file in that pool and random read for 10 sec.
-#	3. Read the amount of log blocks written from the header of the
-#		L2ARC device.
-#	4. Offline the L2ARC device.
-#	5. Online the L2ARC device.
-#	6. Read the amount of log blocks rebuilt in arcstats and compare to
-#		(3).
-#	7. Create another random file in that pool and random read for 10 sec.
-#	8. Read the amount of log blocks written from the header of the
-#		L2ARC device.
-#	9. Offline the L2ARC device.
-#	10. Online the L2ARC device.
-#	11. Read the amount of log blocks rebuilt in arcstats and compare to
-#		(8).
-#	12. Check if the amount of log blocks on the cache device has
-#		increased.
-#	13. Export the pool.
-#	14. Read the amount of log blocks on the cache device.
-#	15. Import the pool.
-#	16. Read the amount of log blocks rebuilt in arcstats and compare to
-#		(14).
-#	17. Check if the labels of the L2ARC device are intact.
-#
-
-verify_runnable "global"
-
-log_assert "Off/onlining an L2ARC device restores all written blocks , vdev present."
-
-function cleanup
-{
-	if poolexists $TESTPOOL ; then
-		destroy_pool $TESTPOOL
-	fi
-
-	log_must set_tunable32 L2ARC_NOPREFETCH $noprefetch
-}
-log_onexit cleanup
-
-# L2ARC_NOPREFETCH is set to 0 to let L2ARC handle prefetches
-typeset noprefetch=$(get_tunable L2ARC_NOPREFETCH)
-log_must set_tunable32 L2ARC_NOPREFETCH 0
-
-typeset fill_mb=400
-typeset cache_sz=$(( 3 * $fill_mb ))
-export FILE_SIZE=$(( floor($fill_mb / $NUMJOBS) ))M
-
-log_must truncate -s ${cache_sz}M $VDEV_CACHE
-
-log_must zpool create -f $TESTPOOL $VDEV cache $VDEV_CACHE
-
-log_must fio $FIO_SCRIPTS/mkfiles.fio
-log_must fio $FIO_SCRIPTS/random_reads.fio
-
-arcstat_quiescence_noecho l2_size
-log_must zpool offline $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-typeset l2_dh_log_blk1=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
-	awk '{print $2}')
-typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
-
-log_must zpool online $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-typeset l2_rebuild_log_blk_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
-
-log_must test $l2_dh_log_blk1 -eq $(( $l2_rebuild_log_blk_end - \
-	$l2_rebuild_log_blk_start ))
-log_must test $l2_dh_log_blk1 -gt 0
-
-log_must fio $FIO_SCRIPTS/mkfiles.fio
-log_must fio $FIO_SCRIPTS/random_reads.fio
-
-arcstat_quiescence_noecho l2_size
-log_must zpool offline $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-typeset l2_dh_log_blk2=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
-	awk '{print $2}')
-typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
-
-log_must zpool online $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-typeset l2_rebuild_log_blk_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
-
-log_must test $l2_dh_log_blk2 -eq $(( $l2_rebuild_log_blk_end - \
-	$l2_rebuild_log_blk_start ))
-log_must test $l2_dh_log_blk2 -gt $l2_dh_log_blk1
-
-log_must zpool export $TESTPOOL
-arcstat_quiescence_noecho l2_feeds
-
-typeset l2_dh_log_blk3=$(zdb -l $VDEV_CACHE | grep log_blk_count | \
-	awk '{print $2}')
-typeset l2_rebuild_log_blk_start=$(get_arcstat l2_rebuild_log_blks)
-
-log_must zpool import -d $VDIR $TESTPOOL
-arcstat_quiescence_noecho l2_size
-
-typeset l2_rebuild_log_blk_end=$(arcstat_quiescence_echo l2_rebuild_log_blks)
-
-log_must test $l2_dh_log_blk3 -eq $(( $l2_rebuild_log_blk_end - \
-	$l2_rebuild_log_blk_start ))
-log_must test $l2_dh_log_blk3 -gt 0
-
-log must zpool offline $TESTPOOL $VDEV_CACHE
-arcstat_quiescence_noecho l2_size
-
-log_must zdb -lq $VDEV_CACHE
-
-log_must zpool destroy -f $TESTPOOL
-
-log_pass "Off/onlining an L2ARC device restores all written blocks, vdev present."