Restore FreeBSD sysctl processing for arc.min and arc.max

Before OpenZFS 2.0, trying to set the FreeBSD sysctl vfs.zfs.arc_max
to a disallowed value would return an error.
Since the switch, it instead only generates WARN_IF_TUNING_IGNORED

Keep the ability to set the sysctl's specifically to 0, even though
that is less than the minimum, because some tests depend on this.

Also lost, was the ability to set vfs.zfs.arc_max to a value less
than the default vfs.zfs.arc_min at boot time. Restore this as well.

Reviewed-by: Tony Nguyen <tony.nguyen@delphix.com>
Reviewed-by: Ryan Moeller <ryan@ixsystems.com>
Signed-off-by: Allan Jude <allan@klarasystems.com>
Closes #12161

6 files changed, 98 insertions, 6 deletions

diff --git a/include/os/freebsd/spl/sys/mod_os.h b/include/os/freebsd/spl/sys/mod_os.h
index 5b3b3271e..5695abee7 100644
--- a/include/os/freebsd/spl/sys/mod_os.h
+++ b/include/os/freebsd/spl/sys/mod_os.h
@@ -62,6 +62,12 @@
 #define	param_set_arc_long_args(var) \
     CTLTYPE_ULONG, &var, 0, param_set_arc_long, "LU"
 
+#define	param_set_arc_min_args(var) \
+    CTLTYPE_ULONG, &var, 0, param_set_arc_min, "LU"
+
+#define	param_set_arc_max_args(var) \
+    CTLTYPE_ULONG, &var, 0, param_set_arc_max, "LU"
+
 #define	param_set_arc_int_args(var) \
     CTLTYPE_INT, &var, 0, param_set_arc_int, "I"
 
diff --git a/include/sys/arc.h b/include/sys/arc.h
index ef07a657f..a3241f368 100644
--- a/include/sys/arc.h
+++ b/include/sys/arc.h
@@ -46,6 +46,13 @@ extern "C" {
  */
 #define	ARC_EVICT_ALL	UINT64_MAX
 
+/*
+ * ZFS gets very unhappy when the maximum ARC size is smaller than the maximum
+ * block size and a larger block is written.  To leave some safety margin, we
+ * limit the minimum for zfs_arc_max to the maximium transaction size.
+ */
+#define	MIN_ARC_MAX	DMU_MAX_ACCESS
+
 #define	HDR_SET_LSIZE(hdr, x) do { \
 	ASSERT(IS_P2ALIGNED(x, 1U << SPA_MINBLOCKSHIFT)); \
 	(hdr)->b_lsize = ((x) >> SPA_MINBLOCKSHIFT); \
diff --git a/include/sys/arc_impl.h b/include/sys/arc_impl.h
index 747100a22..d29993f1d 100644
--- a/include/sys/arc_impl.h
+++ b/include/sys/arc_impl.h
@@ -1005,6 +1005,8 @@ extern void arc_unregister_hotplug(void);
 
 extern int param_set_arc_long(ZFS_MODULE_PARAM_ARGS);
 extern int param_set_arc_int(ZFS_MODULE_PARAM_ARGS);
+extern int param_set_arc_min(ZFS_MODULE_PARAM_ARGS);
+extern int param_set_arc_max(ZFS_MODULE_PARAM_ARGS);
 
 /* used in zdb.c */
 boolean_t l2arc_log_blkptr_valid(l2arc_dev_t *dev,
diff --git a/module/os/freebsd/zfs/sysctl_os.c b/module/os/freebsd/zfs/sysctl_os.c
index 94124fdcf..5315b6098 100644
--- a/module/os/freebsd/zfs/sysctl_os.c
+++ b/module/os/freebsd/zfs/sysctl_os.c
@@ -144,6 +144,55 @@ extern arc_state_t ARC_l2c_only;
 
 /* arc.c */
 
+int
+param_set_arc_max(SYSCTL_HANDLER_ARGS)
+{
+	uint64_t val;
+	int err;
+
+	val = zfs_arc_max;
+	err = sysctl_handle_long(oidp, &val, 0, req);
+	if (err != 0 || req->newptr == NULL)
+		return (SET_ERROR(err));
+
+	if (val != 0 && (val < MIN_ARC_MAX || val <= arc_c_min ||
+	    val >= arc_all_memory()))
+		return (SET_ERROR(EINVAL));
+
+	zfs_arc_max = val;
+	arc_tuning_update(B_TRUE);
+
+	/* Update the sysctl to the tuned value */
+	if (val != 0)
+		zfs_arc_max = arc_c_max;
+
+	return (0);
+}
+
+int
+param_set_arc_min(SYSCTL_HANDLER_ARGS)
+{
+	uint64_t val;
+	int err;
+
+	val = zfs_arc_min;
+	err = sysctl_handle_64(oidp, &val, 0, req);
+	if (err != 0 || req->newptr == NULL)
+		return (SET_ERROR(err));
+
+	if (val != 0 && (val < 2ULL << SPA_MAXBLOCKSHIFT || val > arc_c_max))
+		return (SET_ERROR(EINVAL));
+
+	zfs_arc_min = val;
+	arc_tuning_update(B_TRUE);
+
+	/* Update the sysctl to the tuned value */
+	if (val != 0)
+		zfs_arc_min = arc_c_min;
+
+	return (0);
+}
+
 /* legacy compat */
 extern uint64_t l2arc_write_max;	/* def max write size */
 extern uint64_t l2arc_write_boost;	/* extra warmup write */
@@ -278,11 +327,11 @@ param_set_arc_int(SYSCTL_HANDLER_ARGS)
 
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_min,
     CTLTYPE_ULONG | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
-    &zfs_arc_min, sizeof (zfs_arc_min), param_set_arc_long, "LU",
+    &zfs_arc_min, sizeof (zfs_arc_min), param_set_arc_min, "LU",
     "min arc size (LEGACY)");
 SYSCTL_PROC(_vfs_zfs, OID_AUTO, arc_max,
     CTLTYPE_ULONG | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
-    &zfs_arc_max, sizeof (zfs_arc_max), param_set_arc_long, "LU",
+    &zfs_arc_max, sizeof (zfs_arc_max), param_set_arc_max, "LU",
     "max arc size (LEGACY)");
 
 /* dbuf.c */
diff --git a/module/os/linux/zfs/arc_os.c b/module/os/linux/zfs/arc_os.c
index 415cfc281..c3e889d54 100644
--- a/module/os/linux/zfs/arc_os.c
+++ b/module/os/linux/zfs/arc_os.c
@@ -372,6 +372,18 @@ param_set_arc_long(const char *buf, zfs_kernel_param_t *kp)
 }
 
 int
+param_set_arc_min(const char *buf, zfs_kernel_param_t *kp)
+{
+	return (param_set_arc_long(buf, kp));
+}
+
+int
+param_set_arc_max(const char *buf, zfs_kernel_param_t *kp)
+{
+	return (param_set_arc_long(buf, kp));
+}
+
+int
 param_set_arc_int(const char *buf, zfs_kernel_param_t *kp)
 {
 	int error;
diff --git a/module/zfs/arc.c b/module/zfs/arc.c
index 0a72d8708..70d122130 100644
--- a/module/zfs/arc.c
+++ b/module/zfs/arc.c
@@ -7503,7 +7503,7 @@ arc_tuning_update(boolean_t verbose)
 
 	/* Valid range: 64M - <all physical memory> */
 	if ((zfs_arc_max) && (zfs_arc_max != arc_c_max) &&
-	    (zfs_arc_max >= 64 << 20) && (zfs_arc_max < allmem) &&
+	    (zfs_arc_max >= MIN_ARC_MAX) && (zfs_arc_max < allmem) &&
 	    (zfs_arc_max > arc_c_min)) {
 		arc_c_max = zfs_arc_max;
 		arc_c = MIN(arc_c, arc_c_max);
@@ -7898,7 +7898,23 @@ arc_init(void)
 
 	arc_set_limits(allmem);
 
-#ifndef _KERNEL
+#ifdef _KERNEL
+	/*
+	 * If zfs_arc_max is non-zero at init, meaning it was set in the kernel
+	 * environment before the module was loaded, don't block setting the
+	 * maximum because it is less than arc_c_min, instead, reset arc_c_min
+	 * to a lower value.
+	 * zfs_arc_min will be handled by arc_tuning_update().
+	 */
+	if (zfs_arc_max != 0 && zfs_arc_max >= MIN_ARC_MAX &&
+	    zfs_arc_max < allmem) {
+		arc_c_max = zfs_arc_max;
+		if (arc_c_min >= arc_c_max) {
+			arc_c_min = MAX(zfs_arc_max / 2,
+			    2ULL << SPA_MAXBLOCKSHIFT);
+		}
+	}
+#else
 	/*
 	 * In userland, there's only the memory pressure that we artificially
 	 * create (see arc_available_memory()).  Don't let arc_c get too
@@ -10959,10 +10975,10 @@ EXPORT_SYMBOL(arc_add_prune_callback);
 EXPORT_SYMBOL(arc_remove_prune_callback);
 
 /* BEGIN CSTYLED */
-ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min, param_set_arc_long,
+ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, min, param_set_arc_min,
 	param_get_long, ZMOD_RW, "Min arc size");
 
-ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, max, param_set_arc_long,
+ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, max, param_set_arc_max,
 	param_get_long, ZMOD_RW, "Max arc size");
 
 ZFS_MODULE_PARAM_CALL(zfs_arc, zfs_arc_, meta_limit, param_set_arc_long,