Run arc_evict thread at higher priority

Run arc_evict thread at higher priority, nice=0, to give it more CPU
time which can improve performance for workload with high ARC evict
activities.

On mixed read/write and sequential read workloads, I've seen between
10-40% better performance.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Reviewed-by: George Melikov <mail@gmelikov.ru>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Signed-off-by: Tony Nguyen <tony.nguyen@delphix.com>
Closes #12397

5 files changed, 22 insertions, 15 deletions

diff --git a/include/sys/zthr.h b/include/sys/zthr.h
index ae8c57e9e..19be89eee 100644
--- a/include/sys/zthr.h
+++ b/include/sys/zthr.h
@@ -25,10 +25,11 @@ typedef void (zthr_func_t)(void *, zthr_t *);
 typedef boolean_t (zthr_checkfunc_t)(void *, zthr_t *);
 
 extern zthr_t *zthr_create(const char *zthr_name,
-    zthr_checkfunc_t checkfunc, zthr_func_t *func, void *arg);
+    zthr_checkfunc_t checkfunc, zthr_func_t *func, void *arg,
+	pri_t pri);
 extern zthr_t *zthr_create_timer(const char *zthr_name,
     zthr_checkfunc_t *checkfunc, zthr_func_t *func, void *arg,
-	hrtime_t nano_wait);
+	hrtime_t nano_wait, pri_t pri);
 extern void zthr_destroy(zthr_t *t);
 
 extern void zthr_wakeup(zthr_t *t);
diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 84f73e987..2e02ae9c8 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -7956,9 +7956,9 @@ arc_init(void)
 	}
 
 	arc_evict_zthr = zthr_create("arc_evict",
-	    arc_evict_cb_check, arc_evict_cb, NULL);
+	    arc_evict_cb_check, arc_evict_cb, NULL, defclsyspri);
 	arc_reap_zthr = zthr_create_timer("arc_reap",
-	    arc_reap_cb_check, arc_reap_cb, NULL, SEC2NSEC(1));
+	    arc_reap_cb_check, arc_reap_cb, NULL, SEC2NSEC(1), minclsyspri);
 
 	arc_warm = B_FALSE;
 
diff --git a/module/zfs/spa.c b/module/zfs/spa.c
index 53e391f38..aeda4b003 100644
--- a/module/zfs/spa.c
+++ b/module/zfs/spa.c
@@ -2610,7 +2610,8 @@ spa_start_livelist_destroy_thread(spa_t *spa)
 	ASSERT3P(spa->spa_livelist_delete_zthr, ==, NULL);
 	spa->spa_livelist_delete_zthr =
 	    zthr_create("z_livelist_destroy",
-	    spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa);
+	    spa_livelist_delete_cb_check, spa_livelist_delete_cb, spa,
+	    minclsyspri);
 }
 
 typedef struct livelist_new_arg {
@@ -2820,7 +2821,7 @@ spa_start_livelist_condensing_thread(spa_t *spa)
 	spa->spa_livelist_condense_zthr =
 	    zthr_create("z_livelist_condense",
 	    spa_livelist_condense_cb_check,
-	    spa_livelist_condense_cb, spa);
+	    spa_livelist_condense_cb, spa, minclsyspri);
 }
 
 static void
@@ -2838,7 +2839,7 @@ spa_spawn_aux_threads(spa_t *spa)
 	spa->spa_checkpoint_discard_zthr =
 	    zthr_create("z_checkpoint_discard",
 	    spa_checkpoint_discard_thread_check,
-	    spa_checkpoint_discard_thread, spa);
+	    spa_checkpoint_discard_thread, spa, minclsyspri);
 }
 
 /*
diff --git a/module/zfs/vdev_indirect.c b/module/zfs/vdev_indirect.c
index 1b05ff03a..8fdfa566b 100644
--- a/module/zfs/vdev_indirect.c
+++ b/module/zfs/vdev_indirect.c
@@ -885,7 +885,7 @@ spa_start_indirect_condensing_thread(spa_t *spa)
 	ASSERT3P(spa->spa_condense_zthr, ==, NULL);
 	spa->spa_condense_zthr = zthr_create("z_indirect_condense",
 	    spa_condense_indirect_thread_check,
-	    spa_condense_indirect_thread, spa);
+	    spa_condense_indirect_thread, spa, minclsyspri);
 }
 
 /*
diff --git a/module/zfs/zthr.c b/module/zfs/zthr.c
index 5ac2e3046..33fdda7b6 100644
--- a/module/zfs/zthr.c
+++ b/module/zfs/zthr.c
@@ -83,10 +83,11 @@
  * can be cancelled while doing work and not while checking for work.
  *
  * To start a zthr:
- *     zthr_t *zthr_pointer = zthr_create(checkfunc, func, args);
+ *     zthr_t *zthr_pointer = zthr_create(checkfunc, func, args,
+ *         pri);
  * or
  *     zthr_t *zthr_pointer = zthr_create_timer(checkfunc, func,
- *         args, max_sleep);
+ *         args, max_sleep, pri);
  *
  * After that you should be able to wakeup, cancel, and resume the
  * zthr from another thread using the zthr_pointer.
@@ -220,6 +221,9 @@ struct zthr {
 	 */
 	hrtime_t	zthr_sleep_timeout;
 
+	/* Thread priority */
+	pri_t		zthr_pri;
+
 	/* consumer-provided callbacks & data */
 	zthr_checkfunc_t	*zthr_checkfunc;
 	zthr_func_t	*zthr_func;
@@ -269,10 +273,10 @@ zthr_procedure(void *arg)
 
 zthr_t *
 zthr_create(const char *zthr_name, zthr_checkfunc_t *checkfunc,
-    zthr_func_t *func, void *arg)
+    zthr_func_t *func, void *arg, pri_t pri)
 {
 	return (zthr_create_timer(zthr_name, checkfunc,
-	    func, arg, (hrtime_t)0));
+	    func, arg, (hrtime_t)0, pri));
 }
 
 /*
@@ -282,7 +286,7 @@ zthr_create(const char *zthr_name, zthr_checkfunc_t *checkfunc,
  */
 zthr_t *
 zthr_create_timer(const char *zthr_name, zthr_checkfunc_t *checkfunc,
-    zthr_func_t *func, void *arg, hrtime_t max_sleep)
+    zthr_func_t *func, void *arg, hrtime_t max_sleep, pri_t pri)
 {
 	zthr_t *t = kmem_zalloc(sizeof (*t), KM_SLEEP);
 	mutex_init(&t->zthr_state_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -296,9 +300,10 @@ zthr_create_timer(const char *zthr_name, zthr_checkfunc_t *checkfunc,
 	t->zthr_arg = arg;
 	t->zthr_sleep_timeout = max_sleep;
 	t->zthr_name = zthr_name;
+	t->zthr_pri = pri;
 
 	t->zthr_thread = thread_create_named(zthr_name, NULL, 0,
-	    zthr_procedure, t, 0, &p0, TS_RUN, minclsyspri);
+	    zthr_procedure, t, 0, &p0, TS_RUN, pri);
 
 	mutex_exit(&t->zthr_state_lock);
 
@@ -423,7 +428,7 @@ zthr_resume(zthr_t *t)
 	 */
 	if (t->zthr_thread == NULL) {
 		t->zthr_thread = thread_create_named(t->zthr_name, NULL, 0,
-		    zthr_procedure, t, 0, &p0, TS_RUN, minclsyspri);
+		    zthr_procedure, t, 0, &p0, TS_RUN, t->zthr_pri);
 	}
 
 	mutex_exit(&t->zthr_state_lock);