zinject: Introduce waiting on injection events

This is useful for reproducing races during the sync pipeline

Signed-off-by: Robert Evans <[email protected]>

Author: rrevans

cmd/zinject/zinject.c

@@ -107,6 +107,7 @@
  * 	zinject
  * 	zinject <-a | -u pool>
  * 	zinject -c <id|all>
+ * 	zinject -w <state|0> [-W <delay>]
  * 	zinject -E <delay> [-a] [-m] [-f freq] [-l level] [-r range]
  *	    [-T iotype] [-t type object | -b bookmark pool]
  * 	zinject [-q] <-t type> [-f freq] [-u] [-a] [-m] [-e errno] [-l level]
@@ -119,6 +120,11 @@
  * The '-c' option will clear the given handler, or all handlers if 'all' is
  * specified.
  *
+ * The '-w' flag waits until an injection event occurs. Wait calls accept a
+ * state value to ensure no events are lost. Use '-w 0 -W 0' initially and
+ * then pass the state value printed on stdout to subsequent wait calls.
+ * The optional '-W' flag sets an optional timeout in seconds.
+ *
  * The '-e' option takes a string describing the errno to simulate.  This must
  * be one of 'io', 'checksum', 'decompress', or 'decrypt'.  In most cases this
  * will result in the same behavior, but RAID-Z will produce a different set of
@@ -297,6 +303,14 @@ usage(void)
 	    "\t\tClear the particular record (if given a numeric ID), or\n"
 	    "\t\tall records if 'all' is specified.\n"
 	    "\n"
+	    "\tzinject -w <state|0> [-W delay]\n"
+	    "\n"
+	    "\t\tWait for an injection event to occur.  The 'state' parameter\n"
+	    "\t\tshould be set to zero initially then the value printed to\n"
+	    "\t\tstdout after each call to synchronize with kernel state.\n"
+	    "\t\tThe optional timeout is specified in milliseconds.\n"
+	    "\t\tWaits forever if timeout is omitted.\n"
+	    "\n"
 	    "\tzinject -p <function name> pool\n"
 	    "\t\tInject a panic fault at the specified function. Only \n"
 	    "\t\tfunctions which call spa_vdev_config_exit(), or \n"
@@ -938,6 +952,8 @@ main(int argc, char **argv)
 	int flags = 0;
 	uint32_t dvas = 0;
 	hrtime_t ready_delay = -1;
+	char *wait = NULL;
+	hrtime_t wait_timeout = -1;
 
 	if ((g_zfs = libzfs_init()) == NULL) {
 		(void) fprintf(stderr, "%s\n", libzfs_error_init(errno));
@@ -968,7 +984,7 @@ main(int argc, char **argv)
 	}
 
 	while ((c = getopt(argc, argv,
-	    ":aA:b:C:d:D:E:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:")) != -1) {
+	    ":aA:b:C:d:D:E:f:Fg:qhIc:t:T:l:mr:s:e:uL:p:P:w:W:")) != -1) {
 		switch (c) {
 		case 'a':
 			flags |= ZINJECT_FLUSH_ARC;
@@ -1141,6 +1157,9 @@ main(int argc, char **argv)
 		case 'u':
 			flags |= ZINJECT_UNLOAD_SPA;
 			break;
+		case 'w':
+			wait = optarg;
+			break;
 		case 'E':
 			ready_delay = MSEC2NSEC(strtol(optarg, &end, 10));
 			if (ready_delay <= 0 || *end != '\0') {
@@ -1163,6 +1182,16 @@ main(int argc, char **argv)
 				return (1);
 			}
 			break;
+		case 'W':
+			wait_timeout = MSEC2NSEC(strtol(optarg, &end, 10));
+			if (wait_timeout < 0 || *end != '\0') {
+				(void) fprintf(stderr, "invalid timeout '%s': "
+				    "must be a non-negative integer\n", optarg);
+				usage();
+				libzfs_fini(g_zfs);
+				return (1);
+			}
+			break;
 		case ':':
 			(void) fprintf(stderr, "option -%c requires an "
 			    "operand\n", optopt);
@@ -1184,19 +1213,36 @@ main(int argc, char **argv)
 	if (record.zi_duration != 0 && record.zi_cmd == 0)
 		record.zi_cmd = ZINJECT_IGNORED_WRITES;
 
-	if (cancel != NULL) {
-		/*
-		 * '-c' is invalid with any other options.
-		 */
-		if (raw != NULL || range != NULL || type != TYPE_INVAL ||
-		    level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED ||
-		    record.zi_freq > 0 || dvas != 0 || ready_delay >= 0) {
+	/*
+	 * '-c' and '-w' are invalid with any other options.
+	 */
+	if (raw != NULL || range != NULL || type != TYPE_INVAL ||
+	    level != 0 || record.zi_cmd != ZINJECT_UNINITIALIZED ||
+	    record.zi_freq > 0 || dvas != 0 || ready_delay >= 0) {
+		if (cancel != NULL) {
 			(void) fprintf(stderr, "cancel (-c) incompatible with "
 			    "any other options\n");
 			usage();
 			libzfs_fini(g_zfs);
 			return (2);
 		}
+		if (wait != NULL) {
+			(void) fprintf(stderr, "wait (-w) incompatible with "
+			    "any other options\n");
+			usage();
+			libzfs_fini(g_zfs);
+			return (2);
+		}
+	}
+
+	if (cancel != NULL) {
+		if (wait != NULL) {
+			(void) fprintf(stderr, "cancel (-c) incompatible with "
+			    "wait (-w) option\n");
+			usage();
+			libzfs_fini(g_zfs);
+			return (2);
+		}
 		if (argc != 0) {
 			(void) fprintf(stderr, "extraneous argument to '-c'\n");
 			usage();
@@ -1219,6 +1265,32 @@ main(int argc, char **argv)
 		}
 	}
 
+	if (wait != NULL) {
+		uint64_t state;
+		if (argc != 0) {
+			(void) fprintf(stderr, "extraneous argument to '-w'\n");
+			usage();
+			libzfs_fini(g_zfs);
+			return (2);
+		}
+		state = (uint64_t)strtoull(wait, &end, 10);
+		if (*end != 0) {
+			(void) fprintf(stderr, "invalid state '%s': "
+			    "must be an unsigned integer\n", wait);
+			usage();
+			libzfs_fini(g_zfs);
+			return (1);
+		}
+		error = lzc_wait_inject(&state, wait_timeout);
+		if (error == ETIMEDOUT)
+			(void) printf("wait timeout\n");
+		else if (error != 0)
+			(void) printf("wait failed: %s\n", strerror(error));
+		else
+			(void) printf("%"PRIu64"\n", state);
+		return (error == 0 ? 0 : 1);
+	}
+
 	if (device != NULL) {
 		/*
 		 * Device (-d) injection uses a completely different mechanism

include/libzfs_core.h

@@ -165,6 +165,8 @@ _LIBZFS_CORE_H int lzc_scrub(zfs_ioc_t, const char *, nvlist_t *, nvlist_t **);
 _LIBZFS_CORE_H int lzc_ddt_prune(const char *, zpool_ddt_prune_unit_t,
     uint64_t);
 
+_LIBZFS_CORE_H int lzc_wait_inject(uint64_t *state, hrtime_t timeout);
+
 #ifdef	__cplusplus
 }
 #endif

include/sys/fs/zfs.h

@@ -1566,6 +1566,7 @@ typedef enum zfs_ioc {
 	ZFS_IOC_POOL_SCRUB,			/* 0x5a57 */
 	ZFS_IOC_POOL_PREFETCH,			/* 0x5a58 */
 	ZFS_IOC_DDT_PRUNE,			/* 0x5a59 */
+	ZFS_IOC_WAIT_INJECT,			/* 0x5a5a */
 
 	/*
 	 * Per-platform (Optional) - 8/128 numbers reserved.
@@ -1826,6 +1827,12 @@ typedef enum {
 #define	DDT_PRUNE_UNIT		"ddt_prune_unit"
 #define	DDT_PRUNE_AMOUNT	"ddt_prune_amount"
 
+/*
+ * The following are names used when invoking ZFS_IOC_WAIT_INJECT.
+ */
+#define	WAIT_INJECT_STATE	"state"
+#define	WAIT_INJECT_TIMEOUT	"timeout"
+
 /*
  * Flags for ZFS_IOC_VDEV_SET_STATE
  */

include/sys/zio.h

@@ -704,6 +704,7 @@ extern int zio_inject_fault(char *name, int flags, int *id,
     struct zinject_record *record);
 extern int zio_inject_list_next(int *id, char *name, size_t buflen,
     struct zinject_record *record);
+extern int zio_inject_wait(uint64_t *state, hrtime_t timeout);
 extern int zio_clear_fault(int id);
 extern void zio_handle_panic_injection(spa_t *spa, const char *tag,
     uint64_t type);

lib/libzfs_core/libzfs_core.c

@@ -1994,3 +1994,28 @@ lzc_ddt_prune(const char *pool, zpool_ddt_prune_unit_t unit, uint64_t amount)
 
 	return (error);
 }
+
+/*
+ * Wait for injection events. See zio_inject_wait.
+ */
+int
+lzc_wait_inject(uint64_t *state, hrtime_t timeout)
+{
+	int error;
+
+	nvlist_t *result = NULL;
+	nvlist_t *args = fnvlist_alloc();
+
+	fnvlist_add_uint64(args, WAIT_INJECT_STATE, *state);
+	if (timeout >= 0)
+		VERIFY0(nvlist_add_hrtime(args, WAIT_INJECT_TIMEOUT, timeout));
+
+	error = lzc_ioctl(ZFS_IOC_WAIT_INJECT, NULL, args, &result);
+	if (error == 0)
+		*state = fnvlist_lookup_uint64(result, WAIT_INJECT_STATE);
+
+	fnvlist_free(args);
+	fnvlist_free(result);
+
+	return (error);
+}

man/man8/zinject.8

@@ -61,6 +61,21 @@ Cancel injection records.
 .
 .It Xo
 .Nm zinject
+.Fl w Ar state Ns | Ns Sy 0
+.Op Fl W Ar delay
+.Xc
+Wait until an injection event occurs.
+The
+.Ar state
+parameter synchronizes with kernel state and
+should be set to 0 for first call with zero timeout,
+then the value printed to stdout after each wait.
+.Fl W Ar delay
+sets an optional timeout in milliseconds.
+If omitted, waits forever.
+.
+.It Xo
+.Nm zinject
 .Fl d Ar vdev
 .Fl A Sy degrade Ns | Ns Sy fault
 .Ar pool

module/zfs/zfs_ioctl.c

@@ -6094,6 +6094,41 @@ zfs_ioc_inject_list_next(zfs_cmd_t *zc)
 	return (error);
 }
 
+/*
+ * Waits for an injection event to occur (event injected or handler add/remove).
+ * innvl: {
+ *   "state": wait state token
+ *   "timeout": how long to wait in nanoseconds
+ * }
+ * outnvl: {
+ *   "state": wait state token
+ * }
+ * Returns 0, EINTR, or ETIMEDOUT.
+ */
+static const zfs_ioc_key_t zfs_keys_wait_inject[] = {
+	{"state",	DATA_TYPE_UINT64,	ZK_OPTIONAL},
+	{"timeout",	DATA_TYPE_HRTIME,	ZK_OPTIONAL},
+};
+
+static int
+zfs_ioc_wait_inject(const char *name, nvlist_t *innvl, nvlist_t *outnvl)
+{
+	(void) name;
+	uint64_t state;
+	hrtime_t timeout;
+	int error;
+
+	if (nvlist_lookup_uint64(innvl, "state", &state) != 0)
+		state = 0;
+	if (nvlist_lookup_hrtime(innvl, "timeout", &timeout) != 0)
+		timeout = TIME_MAX;
+
+	error = zio_inject_wait(&state, timeout);
+	fnvlist_add_uint64(outnvl, "state", state);
+
+	return (error);
+}
+
 static int
 zfs_ioc_error_log(zfs_cmd_t *zc)
 {
@@ -7687,6 +7722,10 @@ zfs_ioctl_init(void)
 	    zfs_ioc_clear_fault, zfs_secpolicy_inject);
 	zfs_ioctl_register_pool_meta(ZFS_IOC_INJECT_LIST_NEXT,
 	    zfs_ioc_inject_list_next, zfs_secpolicy_inject);
+	zfs_ioctl_register("wait_inject", ZFS_IOC_WAIT_INJECT,
+	    zfs_ioc_wait_inject, zfs_secpolicy_inject,
+	    NO_NAME, POOL_CHECK_NONE, B_FALSE, B_FALSE,
+	    zfs_keys_wait_inject, ARRAY_SIZE(zfs_keys_wait_inject));
 
 	/*
 	 * pool destroy, and export don't log the history as part of