4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2016 Actifio, Inc. All rights reserved.
34 #include <sys/signal.h>
37 #include <sys/processor.h>
38 #include <sys/zfs_context.h>
39 #include <sys/rrwlock.h>
40 #include <sys/utsname.h>
42 #include <sys/systeminfo.h>
43 #include <zfs_fletcher.h>
44 #include <sys/crypto/icp.h>
47 * Emulation of kernel services in userland.
52 vnode_t *rootdir = (vnode_t *)0xabcd1234;
53 char hw_serial[HW_HOSTID_LEN];
54 struct utsname hw_utsname;
55 vmem_t *zio_arena = NULL;
57 /* If set, all blocks read will be copied to the specified directory. */
58 char *vn_dumpdir = NULL;
60 /* this only exists to have its address taken */
64 * =========================================================================
66 * =========================================================================
68 * TS_STACK_MIN is dictated by the minimum allowed pthread stack size. While
69 * TS_STACK_MAX is somewhat arbitrary, it was selected to be large enough for
70 * the expected stack depth while small enough to avoid exhausting address
71 * space with high thread counts.
73 #define TS_STACK_MIN MAX(PTHREAD_STACK_MIN, 32768)
74 #define TS_STACK_MAX (256 * 1024)
78 zk_thread_create(void (*func)(void *), void *arg, size_t stksize, int state)
83 int detachstate = PTHREAD_CREATE_DETACHED;
85 VERIFY0(pthread_attr_init(&attr));
87 if (state & TS_JOINABLE)
88 detachstate = PTHREAD_CREATE_JOINABLE;
90 VERIFY0(pthread_attr_setdetachstate(&attr, detachstate));
93 * We allow the default stack size in user space to be specified by
94 * setting the ZFS_STACK_SIZE environment variable. This allows us
95 * the convenience of observing and debugging stack overruns in
96 * user space. Explicitly specified stack sizes will be honored.
97 * The usage of ZFS_STACK_SIZE is discussed further in the
98 * ENVIRONMENT VARIABLES sections of the ztest(1) man page.
101 stkstr = getenv("ZFS_STACK_SIZE");
104 stksize = TS_STACK_MAX;
106 stksize = MAX(atoi(stkstr), TS_STACK_MIN);
109 VERIFY3S(stksize, >, 0);
110 stksize = P2ROUNDUP(MAX(stksize, TS_STACK_MIN), PAGESIZE);
113 * If this ever fails, it may be because the stack size is not a
114 * multiple of system page size.
116 VERIFY0(pthread_attr_setstacksize(&attr, stksize));
117 VERIFY0(pthread_attr_setguardsize(&attr, PAGESIZE));
119 VERIFY0(pthread_create(&tid, &attr, (void *(*)(void *))func, arg));
120 VERIFY0(pthread_attr_destroy(&attr));
122 return ((void *)(uintptr_t)tid);
126 * =========================================================================
128 * =========================================================================
132 kstat_create(const char *module, int instance, const char *name,
133 const char *class, uchar_t type, ulong_t ndata, uchar_t ks_flag)
140 kstat_install(kstat_t *ksp)
145 kstat_delete(kstat_t *ksp)
150 kstat_waitq_enter(kstat_io_t *kiop)
155 kstat_waitq_exit(kstat_io_t *kiop)
160 kstat_runq_enter(kstat_io_t *kiop)
165 kstat_runq_exit(kstat_io_t *kiop)
170 kstat_waitq_to_runq(kstat_io_t *kiop)
175 kstat_runq_back_to_waitq(kstat_io_t *kiop)
179 kstat_set_raw_ops(kstat_t *ksp,
180 int (*headers)(char *buf, size_t size),
181 int (*data)(char *buf, size_t size, void *data),
182 void *(*addr)(kstat_t *ksp, loff_t index))
186 * =========================================================================
188 * =========================================================================
192 mutex_init(kmutex_t *mp, char *name, int type, void *cookie)
194 VERIFY0(pthread_mutex_init(&mp->m_lock, NULL));
195 memset(&mp->m_owner, 0, sizeof (pthread_t));
199 mutex_destroy(kmutex_t *mp)
201 VERIFY0(pthread_mutex_destroy(&mp->m_lock));
205 mutex_enter(kmutex_t *mp)
207 VERIFY0(pthread_mutex_lock(&mp->m_lock));
208 mp->m_owner = pthread_self();
212 mutex_tryenter(kmutex_t *mp)
216 error = pthread_mutex_trylock(&mp->m_lock);
218 mp->m_owner = pthread_self();
221 VERIFY3S(error, ==, EBUSY);
227 mutex_exit(kmutex_t *mp)
229 memset(&mp->m_owner, 0, sizeof (pthread_t));
230 VERIFY0(pthread_mutex_unlock(&mp->m_lock));
234 * =========================================================================
236 * =========================================================================
240 rw_init(krwlock_t *rwlp, char *name, int type, void *arg)
242 VERIFY0(pthread_rwlock_init(&rwlp->rw_lock, NULL));
243 rwlp->rw_readers = 0;
248 rw_destroy(krwlock_t *rwlp)
250 VERIFY0(pthread_rwlock_destroy(&rwlp->rw_lock));
254 rw_enter(krwlock_t *rwlp, krw_t rw)
256 if (rw == RW_READER) {
257 VERIFY0(pthread_rwlock_rdlock(&rwlp->rw_lock));
258 atomic_inc_uint(&rwlp->rw_readers);
260 VERIFY0(pthread_rwlock_wrlock(&rwlp->rw_lock));
261 rwlp->rw_owner = pthread_self();
266 rw_exit(krwlock_t *rwlp)
268 if (RW_READ_HELD(rwlp))
269 atomic_dec_uint(&rwlp->rw_readers);
273 VERIFY0(pthread_rwlock_unlock(&rwlp->rw_lock));
277 rw_tryenter(krwlock_t *rwlp, krw_t rw)
282 error = pthread_rwlock_tryrdlock(&rwlp->rw_lock);
284 error = pthread_rwlock_trywrlock(&rwlp->rw_lock);
288 atomic_inc_uint(&rwlp->rw_readers);
290 rwlp->rw_owner = pthread_self();
295 VERIFY3S(error, ==, EBUSY);
302 zone_get_hostid(void *zonep)
305 * We're emulating the system's hostid in userland.
307 return (strtoul(hw_serial, NULL, 10));
311 rw_tryupgrade(krwlock_t *rwlp)
317 * =========================================================================
318 * condition variables
319 * =========================================================================
323 cv_init(kcondvar_t *cv, char *name, int type, void *arg)
325 VERIFY0(pthread_cond_init(cv, NULL));
329 cv_destroy(kcondvar_t *cv)
331 VERIFY0(pthread_cond_destroy(cv));
335 cv_wait(kcondvar_t *cv, kmutex_t *mp)
337 memset(&mp->m_owner, 0, sizeof (pthread_t));
338 VERIFY0(pthread_cond_wait(cv, &mp->m_lock));
339 mp->m_owner = pthread_self();
343 cv_timedwait(kcondvar_t *cv, kmutex_t *mp, clock_t abstime)
350 delta = abstime - ddi_get_lbolt();
354 VERIFY(gettimeofday(&tv, NULL) == 0);
356 ts.tv_sec = tv.tv_sec + delta / hz;
357 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % hz) * (NANOSEC / hz);
358 if (ts.tv_nsec >= NANOSEC) {
360 ts.tv_nsec -= NANOSEC;
363 memset(&mp->m_owner, 0, sizeof (pthread_t));
364 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
365 mp->m_owner = pthread_self();
367 if (error == ETIMEDOUT)
377 cv_timedwait_hires(kcondvar_t *cv, kmutex_t *mp, hrtime_t tim, hrtime_t res,
385 ASSERT(flag == 0 || flag == CALLOUT_FLAG_ABSOLUTE);
388 if (flag & CALLOUT_FLAG_ABSOLUTE)
389 delta -= gethrtime();
394 VERIFY0(gettimeofday(&tv, NULL));
396 ts.tv_sec = tv.tv_sec + delta / NANOSEC;
397 ts.tv_nsec = tv.tv_usec * NSEC_PER_USEC + (delta % NANOSEC);
398 if (ts.tv_nsec >= NANOSEC) {
400 ts.tv_nsec -= NANOSEC;
403 memset(&mp->m_owner, 0, sizeof (pthread_t));
404 error = pthread_cond_timedwait(cv, &mp->m_lock, &ts);
405 mp->m_owner = pthread_self();
407 if (error == ETIMEDOUT)
416 cv_signal(kcondvar_t *cv)
418 VERIFY0(pthread_cond_signal(cv));
422 cv_broadcast(kcondvar_t *cv)
424 VERIFY0(pthread_cond_broadcast(cv));
428 * =========================================================================
430 * =========================================================================
433 * Note: for the xxxat() versions of these functions, we assume that the
434 * starting vp is always rootdir (which is true for spa_directory.c, the only
435 * ZFS consumer of these interfaces). We assert this is true, and then emulate
436 * them by adding '/' in front of the path.
441 vn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
451 realpath = umem_alloc(MAXPATHLEN, UMEM_NOFAIL);
454 * If we're accessing a real disk from userland, we need to use
455 * the character interface to avoid caching. This is particularly
456 * important if we're trying to look at a real in-kernel storage
457 * pool from userland, e.g. via zdb, because otherwise we won't
458 * see the changes occurring under the segmap cache.
459 * On the other hand, the stupid character device returns zero
460 * for its size. So -- gag -- we open the block device to get
461 * its size, and remember it for subsequent VOP_GETATTR().
463 #if defined(__sun__) || defined(__sun)
464 if (strncmp(path, "/dev/", 5) == 0) {
469 fd = open64(path, O_RDONLY);
475 if (fstat64(fd, &st) == -1) {
482 (void) sprintf(realpath, "%s", path);
483 dsk = strstr(path, "/dsk/");
485 (void) sprintf(realpath + (dsk - path) + 1, "r%s",
488 (void) sprintf(realpath, "%s", path);
489 if (!(flags & FCREAT) && stat64(realpath, &st) == -1) {
496 if (!(flags & FCREAT) && S_ISBLK(st.st_mode)) {
500 /* We shouldn't be writing to block devices in userspace */
501 VERIFY(!(flags & FWRITE));
505 old_umask = umask(0);
508 * The construct 'flags - FREAD' conveniently maps combinations of
509 * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
511 fd = open64(realpath, flags - FREAD, mode);
519 (void) umask(old_umask);
521 if (vn_dumpdir != NULL) {
522 char *dumppath = umem_zalloc(MAXPATHLEN, UMEM_NOFAIL);
523 (void) snprintf(dumppath, MAXPATHLEN,
524 "%s/%s", vn_dumpdir, basename(realpath));
525 dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
526 umem_free(dumppath, MAXPATHLEN);
539 if (fstat64_blk(fd, &st) == -1) {
547 (void) fcntl(fd, F_SETFD, FD_CLOEXEC);
549 *vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
552 vp->v_size = st.st_size;
553 vp->v_path = spa_strdup(path);
554 vp->v_dump_fd = dump_fd;
561 vn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
562 int x3, vnode_t *startvp, int fd)
564 char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
567 ASSERT(startvp == rootdir);
568 (void) sprintf(realpath, "/%s", path);
570 /* fd ignored for now, need if want to simulate nbmand support */
571 ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
573 umem_free(realpath, strlen(path) + 2);
580 vn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
581 int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
583 ssize_t rc, done = 0, split;
585 if (uio == UIO_READ) {
586 rc = pread64(vp->v_fd, addr, len, offset);
587 if (vp->v_dump_fd != -1 && rc != -1) {
589 status = pwrite64(vp->v_dump_fd, addr, rc, offset);
590 ASSERT(status != -1);
594 * To simulate partial disk writes, we split writes into two
595 * system calls so that the process can be killed in between.
597 int sectors = len >> SPA_MINBLOCKSHIFT;
598 split = (sectors > 0 ? rand() % sectors : 0) <<
600 rc = pwrite64(vp->v_fd, addr, split, offset);
603 rc = pwrite64(vp->v_fd, (char *)addr + split,
604 len - split, offset + split);
609 if (rc == -1 && errno == EINVAL) {
611 * Under Linux, this most likely means an alignment issue
612 * (memory or disk) due to O_DIRECT, so we abort() in order to
613 * catch the offender.
624 *residp = len - done;
625 else if (done != len)
631 vn_close(vnode_t *vp)
634 if (vp->v_dump_fd != -1)
635 close(vp->v_dump_fd);
636 spa_strfree(vp->v_path);
637 umem_free(vp, sizeof (vnode_t));
641 * At a minimum we need to update the size since vdev_reopen()
642 * will no longer call vn_openat().
645 fop_getattr(vnode_t *vp, vattr_t *vap)
650 if (fstat64_blk(vp->v_fd, &st) == -1) {
656 vap->va_size = st.st_size;
661 * =========================================================================
662 * Figure out which debugging statements to print
663 * =========================================================================
666 static char *dprintf_string;
667 static int dprintf_print_all;
670 dprintf_find_string(const char *string)
672 char *tmp_str = dprintf_string;
673 int len = strlen(string);
676 * Find out if this is a string we want to print.
677 * String format: file1.c,function_name1,file2.c,file3.c
680 while (tmp_str != NULL) {
681 if (strncmp(tmp_str, string, len) == 0 &&
682 (tmp_str[len] == ',' || tmp_str[len] == '\0'))
684 tmp_str = strchr(tmp_str, ',');
686 tmp_str++; /* Get rid of , */
692 dprintf_setup(int *argc, char **argv)
697 * Debugging can be specified two ways: by setting the
698 * environment variable ZFS_DEBUG, or by including a
699 * "debug=..." argument on the command line. The command
700 * line setting overrides the environment variable.
703 for (i = 1; i < *argc; i++) {
704 int len = strlen("debug=");
705 /* First look for a command line argument */
706 if (strncmp("debug=", argv[i], len) == 0) {
707 dprintf_string = argv[i] + len;
708 /* Remove from args */
709 for (j = i; j < *argc; j++)
716 if (dprintf_string == NULL) {
717 /* Look for ZFS_DEBUG environment variable */
718 dprintf_string = getenv("ZFS_DEBUG");
722 * Are we just turning on all debugging?
724 if (dprintf_find_string("on"))
725 dprintf_print_all = 1;
727 if (dprintf_string != NULL)
728 zfs_flags |= ZFS_DEBUG_DPRINTF;
732 * =========================================================================
734 * =========================================================================
737 __dprintf(const char *file, const char *func, int line, const char *fmt, ...)
743 * Get rid of annoying "../common/" prefix to filename.
745 newfile = strrchr(file, '/');
746 if (newfile != NULL) {
747 newfile = newfile + 1; /* Get rid of leading / */
752 if (dprintf_print_all ||
753 dprintf_find_string(newfile) ||
754 dprintf_find_string(func)) {
755 /* Print out just the function name if requested */
757 if (dprintf_find_string("pid"))
758 (void) printf("%d ", getpid());
759 if (dprintf_find_string("tid"))
760 (void) printf("%u ", (uint_t)pthread_self());
761 if (dprintf_find_string("cpu"))
762 (void) printf("%u ", getcpuid());
763 if (dprintf_find_string("time"))
764 (void) printf("%llu ", gethrtime());
765 if (dprintf_find_string("long"))
766 (void) printf("%s, line %d: ", newfile, line);
767 (void) printf("%s: ", func);
769 (void) vprintf(fmt, adx);
776 * =========================================================================
777 * cmn_err() and panic()
778 * =========================================================================
780 static char ce_prefix[CE_IGNORE][10] = { "", "NOTICE: ", "WARNING: ", "" };
781 static char ce_suffix[CE_IGNORE][2] = { "", "\n", "\n", "" };
784 vpanic(const char *fmt, va_list adx)
786 (void) fprintf(stderr, "error: ");
787 (void) vfprintf(stderr, fmt, adx);
788 (void) fprintf(stderr, "\n");
790 abort(); /* think of it as a "user-level crash dump" */
794 panic(const char *fmt, ...)
804 vcmn_err(int ce, const char *fmt, va_list adx)
808 if (ce != CE_NOTE) { /* suppress noise in userland stress testing */
809 (void) fprintf(stderr, "%s", ce_prefix[ce]);
810 (void) vfprintf(stderr, fmt, adx);
811 (void) fprintf(stderr, "%s", ce_suffix[ce]);
817 cmn_err(int ce, const char *fmt, ...)
822 vcmn_err(ce, fmt, adx);
827 * =========================================================================
829 * =========================================================================
832 kobj_open_file(char *name)
837 /* set vp as the _fd field of the file */
838 if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
840 return ((void *)-1UL);
842 file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
843 file->_fd = (intptr_t)vp;
848 kobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
852 if (vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
853 UIO_SYSSPACE, 0, 0, 0, &resid) != 0)
856 return (size - resid);
860 kobj_close_file(struct _buf *file)
862 vn_close((vnode_t *)file->_fd);
863 umem_free(file, sizeof (struct _buf));
867 kobj_get_filesize(struct _buf *file, uint64_t *size)
870 vnode_t *vp = (vnode_t *)file->_fd;
872 if (fstat64(vp->v_fd, &st) == -1) {
881 * =========================================================================
883 * =========================================================================
889 (void) poll(0, 0, ticks * (1000 / hz));
893 * Find highest one bit set.
894 * Returns bit number + 1 of highest bit that is set, otherwise returns 0.
895 * The __builtin_clzll() function is supported by both GCC and Clang.
898 highbit64(uint64_t i)
903 return (NBBY * sizeof (uint64_t) - __builtin_clzll(i));
907 * Find lowest one bit set.
908 * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
909 * The __builtin_ffsll() function is supported by both GCC and Clang.
917 return (__builtin_ffsll(i));
920 char *random_path = "/dev/random";
921 char *urandom_path = "/dev/urandom";
922 static int random_fd = -1, urandom_fd = -1;
927 VERIFY((random_fd = open(random_path, O_RDONLY)) != -1);
928 VERIFY((urandom_fd = open(urandom_path, O_RDONLY)) != -1);
942 random_get_bytes_common(uint8_t *ptr, size_t len, int fd)
950 bytes = read(fd, ptr, resid);
951 ASSERT3S(bytes, >=, 0);
960 random_get_bytes(uint8_t *ptr, size_t len)
962 return (random_get_bytes_common(ptr, len, random_fd));
966 random_get_pseudo_bytes(uint8_t *ptr, size_t len)
968 return (random_get_bytes_common(ptr, len, urandom_fd));
972 ddi_strtoul(const char *hw_serial, char **nptr, int base, unsigned long *result)
976 *result = strtoul(hw_serial, &end, base);
983 ddi_strtoull(const char *str, char **nptr, int base, u_longlong_t *result)
987 *result = strtoull(str, &end, base);
996 return (&hw_utsname);
1000 * =========================================================================
1001 * kernel emulation setup & teardown
1002 * =========================================================================
1005 umem_out_of_memory(void)
1007 char errmsg[] = "out of memory -- generating core dump\n";
1009 (void) fprintf(stderr, "%s", errmsg);
1015 kernel_init(int mode)
1017 extern uint_t rrw_tsd_key;
1019 umem_nofail_callback(umem_out_of_memory);
1021 physmem = sysconf(_SC_PHYS_PAGES);
1023 dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
1024 (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
1026 (void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
1027 (mode & FWRITE) ? get_system_hostid() : 0);
1031 VERIFY0(uname(&hw_utsname));
1033 system_taskq_init();
1040 tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
1050 system_taskq_fini();
1056 crgetuid(cred_t *cr)
1062 crgetruid(cred_t *cr)
1068 crgetgid(cred_t *cr)
1074 crgetngroups(cred_t *cr)
1080 crgetgroups(cred_t *cr)
1086 zfs_secpolicy_snapshot_perms(const char *name, cred_t *cr)
1092 zfs_secpolicy_rename_perms(const char *from, const char *to, cred_t *cr)
1098 zfs_secpolicy_destroy_perms(const char *name, cred_t *cr)
1104 secpolicy_zfs(const cred_t *cr)
1110 ksid_lookupdomain(const char *dom)
1114 kd = umem_zalloc(sizeof (ksiddomain_t), UMEM_NOFAIL);
1115 kd->kd_name = spa_strdup(dom);
1120 ksiddomain_rele(ksiddomain_t *ksid)
1122 spa_strfree(ksid->kd_name);
1123 umem_free(ksid, sizeof (ksiddomain_t));
1127 kmem_vasprintf(const char *fmt, va_list adx)
1132 va_copy(adx_copy, adx);
1133 VERIFY(vasprintf(&buf, fmt, adx_copy) != -1);
1140 kmem_asprintf(const char *fmt, ...)
1146 VERIFY(vasprintf(&buf, fmt, adx) != -1);
1154 zfs_onexit_fd_hold(int fd, minor_t *minorp)
1162 zfs_onexit_fd_rele(int fd)
1168 zfs_onexit_add_cb(minor_t minor, void (*func)(void *), void *data,
1169 uint64_t *action_handle)
1176 zfs_onexit_del_cb(minor_t minor, uint64_t action_handle, boolean_t fire)
1183 zfs_onexit_cb_data(minor_t minor, uint64_t action_handle, void **data)
1189 spl_fstrans_mark(void)
1191 return ((fstrans_cookie_t)0);
1195 spl_fstrans_unmark(fstrans_cookie_t cookie)
1200 __spl_pf_fstrans_check(void)
1205 void *zvol_tag = "zvol_tag";
1208 zvol_create_minors(spa_t *spa, const char *name, boolean_t async)
1213 zvol_remove_minor(spa_t *spa, const char *name, boolean_t async)
1218 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
1223 zvol_rename_minors(spa_t *spa, const char *oldname, const char *newname,