Merge pull request #215 from truenas/truenas/zfs-2.2.4-staging

Sync with upstream zfs-2.2-4-staging

cmd/arc_summary

@@ -793,18 +793,27 @@ def section_dmu(kstats_dict):
 
     zfetch_stats = isolate_section('zfetchstats', kstats_dict)
 
-    zfetch_access_total = int(zfetch_stats['hits'])+int(zfetch_stats['misses'])
+    zfetch_access_total = int(zfetch_stats['hits']) +\
+        int(zfetch_stats['future']) + int(zfetch_stats['stride']) +\
+        int(zfetch_stats['past']) + int(zfetch_stats['misses'])
 
     prt_1('DMU predictive prefetcher calls:', f_hits(zfetch_access_total))
     prt_i2('Stream hits:',
            f_perc(zfetch_stats['hits'], zfetch_access_total),
            f_hits(zfetch_stats['hits']))
+    future = int(zfetch_stats['future']) + int(zfetch_stats['stride'])
+    prt_i2('Hits ahead of stream:', f_perc(future, zfetch_access_total),
+           f_hits(future))
+    prt_i2('Hits behind stream:',
+           f_perc(zfetch_stats['past'], zfetch_access_total),
+           f_hits(zfetch_stats['past']))
     prt_i2('Stream misses:',
            f_perc(zfetch_stats['misses'], zfetch_access_total),
            f_hits(zfetch_stats['misses']))
     prt_i2('Streams limit reached:',
            f_perc(zfetch_stats['max_streams'], zfetch_stats['misses']),
            f_hits(zfetch_stats['max_streams']))
+    prt_i1('Stream strides:', f_hits(zfetch_stats['stride']))
     prt_i1('Prefetches issued', f_hits(zfetch_stats['io_issued']))
     print()
 

cmd/zdb/zdb.c

@@ -8041,6 +8041,17 @@ dump_mos_leaks(spa_t *spa)
 		}
 	}
 
+	if (spa->spa_brt != NULL) {
+		brt_t *brt = spa->spa_brt;
+		for (uint64_t vdevid = 0; vdevid < brt->brt_nvdevs; vdevid++) {
+			brt_vdev_t *brtvd = &brt->brt_vdevs[vdevid];
+			if (brtvd != NULL && brtvd->bv_initiated) {
+				mos_obj_refd(brtvd->bv_mos_brtvdev);
+				mos_obj_refd(brtvd->bv_mos_entries);
+			}
+		}
+	}
+
 	/*
 	 * Visit all allocated objects and make sure they are referenced.
 	 */

cmd/zed/zed.d/statechange-slot_off.sh

@@ -5,7 +5,7 @@
 #
 # Bad SCSI disks can often "disappear and reappear" causing all sorts of chaos
 # as they flip between FAULTED and ONLINE.  If
-# ZED_POWER_OFF_ENCLOUSRE_SLOT_ON_FAULT is set in zed.rc, and the disk gets
+# ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT is set in zed.rc, and the disk gets
 # FAULTED, then power down the slot via sysfs:
 #
 # /sys/class/enclosure/<enclosure>/<slot>/power_status
@@ -19,7 +19,7 @@
 # Exit codes:
 #   0: slot successfully powered off
 #   1: enclosure not available
-#   2: ZED_POWER_OFF_ENCLOUSRE_SLOT_ON_FAULT disabled
+#   2: ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT disabled
 #   3: vdev was not FAULTED
 #   4: The enclosure sysfs path passed from ZFS does not exist
 #   5: Enclosure slot didn't actually turn off after we told it to
@@ -32,7 +32,7 @@ if [ ! -d /sys/class/enclosure ] ; then
 	exit 1
 fi
 
-if [ "${ZED_POWER_OFF_ENCLOUSRE_SLOT_ON_FAULT}" != "1" ] ; then
+if [ "${ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT}" != "1" ] ; then
 	exit 2
 fi
 

cmd/zed/zed.d/zed-functions.sh

@@ -205,6 +205,10 @@ zed_notify()
     [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
     [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
 
+    zed_notify_ntfy "${subject}" "${pathname}"; rv=$?
+    [ "${rv}" -eq 0 ] && num_success=$((num_success + 1))
+    [ "${rv}" -eq 1 ] && num_failure=$((num_failure + 1))
+
     [ "${num_success}" -gt 0 ] && return 0
     [ "${num_failure}" -gt 0 ] && return 1
     return 2
@@ -527,6 +531,100 @@ zed_notify_pushover()
 }
 
 
+# zed_notify_ntfy (subject, pathname)
+#
+# Send a notification via Ntfy.sh <https://ntfy.sh/>.
+# The ntfy topic (ZED_NTFY_TOPIC) identifies the topic that the notification
+# will be sent to Ntfy.sh server. The ntfy url (ZED_NTFY_URL) defines the
+# self-hosted or provided hosted ntfy service location. The ntfy access token
+# <https://docs.ntfy.sh/publish/#access-tokens> (ZED_NTFY_ACCESS_TOKEN) reprsents an
+# access token that could be used if a topic is read/write protected. If a
+# topic can be written to publicaly, a ZED_NTFY_ACCESS_TOKEN is not required.
+#
+# Requires curl and sed executables to be installed in the standard PATH.
+#
+# References
+#   https://docs.ntfy.sh
+#
+# Arguments
+#   subject: notification subject
+#   pathname: pathname containing the notification message (OPTIONAL)
+#
+# Globals
+#   ZED_NTFY_TOPIC
+#   ZED_NTFY_ACCESS_TOKEN (OPTIONAL)
+#   ZED_NTFY_URL
+#
+# Return
+#   0: notification sent
+#   1: notification failed
+#   2: not configured
+#
+zed_notify_ntfy()
+{
+    local subject="$1"
+    local pathname="${2:-"/dev/null"}"
+    local msg_body
+    local msg_out
+    local msg_err
+
+    [ -n "${ZED_NTFY_TOPIC}" ] || return 2
+    local url="${ZED_NTFY_URL:-"https://ntfy.sh"}/${ZED_NTFY_TOPIC}"
+
+    if [ ! -r "${pathname}" ]; then
+        zed_log_err "ntfy cannot read \"${pathname}\""
+        return 1
+    fi
+
+    zed_check_cmd "curl" "sed" || return 1
+
+    # Read the message body in.
+    #
+    msg_body="$(cat "${pathname}")"
+
+    if [ -z "${msg_body}" ]
+    then
+        msg_body=$subject
+        subject=""
+    fi
+
+    # Send the POST request and check for errors.
+    #
+    if [ -n "${ZED_NTFY_ACCESS_TOKEN}" ]; then
+        msg_out="$( \
+        curl \
+        -u ":${ZED_NTFY_ACCESS_TOKEN}" \
+        -H "Title: ${subject}" \
+        -d "${msg_body}" \
+        -H "Priority: high" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    else
+        msg_out="$( \
+        curl \
+        -H "Title: ${subject}" \
+        -d "${msg_body}" \
+        -H "Priority: high" \
+        "${url}" \
+        2>/dev/null \
+        )"; rv=$?
+    fi
+    if [ "${rv}" -ne 0 ]; then
+        zed_log_err "curl exit=${rv}"
+        return 1
+    fi
+    msg_err="$(echo "${msg_out}" \
+        | sed -n -e 's/.*"errors" *:.*\[\(.*\)\].*/\1/p')"
+    if [ -n "${msg_err}" ]; then
+        zed_log_err "ntfy \"${msg_err}"\"
+        return 1
+    fi
+    return 0
+}
+
+
+
 # zed_rate_limit (tag, [interval])
 #
 # Check whether an event of a given type [tag] has already occurred within the

cmd/zed/zed.d/zed.rc

@@ -146,4 +146,26 @@ ZED_SYSLOG_SUBCLASS_EXCLUDE="history_event"
 # Power off the drive's slot in the enclosure if it becomes FAULTED.  This can
 # help silence misbehaving drives.  This assumes your drive enclosure fully
 # supports slot power control via sysfs.
-#ZED_POWER_OFF_ENCLOUSRE_SLOT_ON_FAULT=1
+#ZED_POWER_OFF_ENCLOSURE_SLOT_ON_FAULT=1
+
+##
+# Ntfy topic
+# This defines which topic will receive the ntfy notification.
+#  <https://docs.ntfy.sh/publish/>
+# Disabled by default; uncomment to enable.
+#ZED_NTFY_TOPIC=""
+
+##
+# Ntfy access token (optional for public topics)
+# This defines an access token which can be used
+# to allow you to authenticate when sending to topics
+# <https://docs.ntfy.sh/publish/#access-tokens>
+# Disabled by default; uncomment to enable.
+#ZED_NTFY_ACCESS_TOKEN=""
+
+##
+# Ntfy Service URL
+# This defines which service the ntfy call will be directed toward
+#  <https://docs.ntfy.sh/install/>
+# https://ntfy.sh by default; uncomment to enable an alternative service url.
+#ZED_NTFY_URL="https://ntfy.sh"

cmd/zfs/zfs_main.c

@@ -3672,15 +3672,25 @@ zfs_do_list(int argc, char **argv)
 
 			for (char *tok; (tok = strsep(&optarg, ",")); ) {
 				static const char *const type_subopts[] = {
-					"filesystem", "volume",
-					"snapshot", "snap",
+					"filesystem",
+					"fs",
+					"volume",
+					"vol",
+					"snapshot",
+					"snap",
 					"bookmark",
-					"all" };
+					"all"
+				};
 				static const int type_types[] = {
-					ZFS_TYPE_FILESYSTEM, ZFS_TYPE_VOLUME,
-					ZFS_TYPE_SNAPSHOT, ZFS_TYPE_SNAPSHOT,
+					ZFS_TYPE_FILESYSTEM,
+					ZFS_TYPE_FILESYSTEM,
+					ZFS_TYPE_VOLUME,
+					ZFS_TYPE_VOLUME,
+					ZFS_TYPE_SNAPSHOT,
+					ZFS_TYPE_SNAPSHOT,
 					ZFS_TYPE_BOOKMARK,
-					ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK };
+					ZFS_TYPE_DATASET | ZFS_TYPE_BOOKMARK
+				};
 
 				for (c = 0; c < ARRAY_SIZE(type_subopts); ++c)
 					if (strcmp(tok, type_subopts[c]) == 0) {

cmd/zpool/zpool_main.c

@@ -2161,6 +2161,7 @@ typedef struct status_cbdata {
 	boolean_t	cb_explain;
 	boolean_t	cb_first;
 	boolean_t	cb_dedup_stats;
+	boolean_t	cb_print_unhealthy;
 	boolean_t	cb_print_status;
 	boolean_t	cb_print_slow_ios;
 	boolean_t	cb_print_vdev_init;
@@ -2357,6 +2358,35 @@ health_str_to_color(const char *health)
 	return (NULL);
 }
 
+/*
+ * Called for each leaf vdev.  Returns 0 if the vdev is healthy.
+ * A vdev is unhealthy if any of the following are true:
+ * 1) there are read, write, or checksum errors,
+ * 2) its state is not ONLINE, or
+ * 3) slow IO reporting was requested (-s) and there are slow IOs.
+ */
+static int
+vdev_health_check_cb(void *hdl_data, nvlist_t *nv, void *data)
+{
+	status_cbdata_t *cb = data;
+	vdev_stat_t *vs;
+	uint_t vsc;
+	(void) hdl_data;
+
+	if (nvlist_lookup_uint64_array(nv, ZPOOL_CONFIG_VDEV_STATS,
+	    (uint64_t **)&vs, &vsc) != 0)
+		return (1);
+
+	if (vs->vs_checksum_errors || vs->vs_read_errors ||
+	    vs->vs_write_errors || vs->vs_state != VDEV_STATE_HEALTHY)
+		return (1);
+
+	if (cb->cb_print_slow_ios && vs->vs_slow_ios)
+		return (1);
+
+	return (0);
+}
+
 /*
  * Print out configuration state as requested by status_callback.
  */
@@ -2375,7 +2405,8 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
 	const char *state;
 	const char *type;
 	const char *path = NULL;
-	const char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL;
+	const char *rcolor = NULL, *wcolor = NULL, *ccolor = NULL,
+	    *scolor = NULL;
 
 	if (nvlist_lookup_nvlist_array(nv, ZPOOL_CONFIG_CHILDREN,
 	    &child, &children) != 0)
@@ -2402,6 +2433,15 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
 			state = gettext("AVAIL");
 	}
 
+	/*
+	 * If '-e' is specified then top-level vdevs and their children
+	 * can be pruned if all of their leaves are healthy.
+	 */
+	if (cb->cb_print_unhealthy && depth > 0 &&
+	    for_each_vdev_in_nvlist(nv, vdev_health_check_cb, cb) == 0) {
+		return;
+	}
+
 	printf_color(health_str_to_color(state),
 	    "\t%*s%-*s  %-8s", depth, "", cb->cb_namewidth - depth,
 	    name, state);
@@ -2416,6 +2456,9 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
 		if (vs->vs_checksum_errors)
 			ccolor = ANSI_RED;
 
+		if (vs->vs_slow_ios)
+			scolor = ANSI_BLUE;
+
 		if (cb->cb_literal) {
 			fputc(' ', stdout);
 			printf_color(rcolor, "%5llu",
@@ -2448,9 +2491,10 @@ print_status_config(zpool_handle_t *zhp, status_cbdata_t *cb, const char *name,
 			}
 
 			if (cb->cb_literal)
-				printf(" %5llu", (u_longlong_t)vs->vs_slow_ios);
+				printf_color(scolor, " %5llu",
+				    (u_longlong_t)vs->vs_slow_ios);
 			else
-				printf(" %5s", rbuf);
+				printf_color(scolor, " %5s", rbuf);
 		}
 		if (cb->cb_print_power) {
 			if (children == 0)  {
@@ -8999,9 +9043,11 @@ status_callback(zpool_handle_t *zhp, void *data)
 				(void) printf(gettext(
 				    "errors: No known data errors\n"));
 			} else if (!cbp->cb_verbose) {
+				color_start(ANSI_RED);
 				(void) printf(gettext("errors: %llu data "
 				    "errors, use '-v' for a list\n"),
 				    (u_longlong_t)nerr);
+				color_end();
 			} else {
 				print_error_log(zhp);
 			}
@@ -9022,6 +9068,7 @@ status_callback(zpool_handle_t *zhp, void *data)
  *              [pool] [interval [count]]
  *
  *	-c CMD	For each vdev, run command CMD
+ *	-e	Display only unhealthy vdevs
  *	-i	Display vdev initialization status.
  *	-g	Display guid for individual vdev name.
  *	-L	Follow links when resolving vdev path name.
@@ -9053,7 +9100,7 @@ zpool_do_status(int argc, char **argv)
 	};
 
 	/* check options */
-	while ((c = getopt_long(argc, argv, "c:igLpPsvxDtT:", long_options,
+	while ((c = getopt_long(argc, argv, "c:eigLpPsvxDtT:", long_options,
 	    NULL)) != -1) {
 		switch (c) {
 		case 'c':
@@ -9080,6 +9127,9 @@ zpool_do_status(int argc, char **argv)
 			}
 			cmd = optarg;
 			break;
+		case 'e':
+			cb.cb_print_unhealthy = B_TRUE;
+			break;
 		case 'i':
 			cb.cb_print_vdev_init = B_TRUE;
 			break;

config/kernel-mm-page-size.m4

@@ -0,0 +1,17 @@
+AC_DEFUN([ZFS_AC_KERNEL_SRC_MM_PAGE_SIZE], [
+	ZFS_LINUX_TEST_SRC([page_size], [
+		#include <linux/mm.h>
+	],[
+		unsigned long s;
+		s = page_size(NULL);
+	])
+])
+AC_DEFUN([ZFS_AC_KERNEL_MM_PAGE_SIZE], [
+	AC_MSG_CHECKING([whether page_size() is available])
+	ZFS_LINUX_TEST_RESULT([page_size], [
+		AC_MSG_RESULT(yes)
+		AC_DEFINE(HAVE_MM_PAGE_SIZE, 1, [page_size() is available])
+	],[
+		AC_MSG_RESULT(no)
+	])
+])