Hello again.
Simon 'corecode' Schubert wrote:
>>> Also, please configure a dumpdev and capture a crash dump.
Before updating I've decided to run %hammer reblock on a slave pfs with
%hammer reblock /mirrorvar
without remounting it before executing a command and got next (reported
with kgdb):
Unread portion of the kernel message buffer:
panic: assertion: cursor->trans->sync_lock_refs > 0 in hammer_recover_cursor
Trace beginning at frame 0xd743e714
panic(d743e738,d743e7b8,d743e89c,d743e814,d743e744) at panic+0x8c
panic(c059aed8,c060299c,c05826fc,d743e7b8,d743e9ac) at panic+0x8c
hammer_recover_cursor(d743e7b8,0,0,5,d743e778) at hammer_recover_cursor+0x2c
hammer_ioc_mirror_write(d743ea84,d2c52550,c2a72998,c2b13fa8,1) at
hammer_ioc_mirror_write+0x928
hammer_ioctl(d2c52550,c0c46808,c2a72998,1,c29a7768) at hammer_ioctl+0x8f2
hammer_vop_ioctl(d743eae0,c0664560,c2ae8b50,d60e9b00,46) at
hammer_vop_ioctl+0x2f
vop_ioctl(c2ae8b50,c2b13fa8,c0c46808,c2a72998,1) at vop_ioctl+0x38
vn_ioctl(d0291148,c0c46808,c2a72998,c29a7768,d0291148) at vn_ioctl+0xdd
mapped_ioctl(3,c0c46808,bfbff794,0,d743ed34) at mapped_ioctl+0x3e1
sys_ioctl(d743ecf0,6,24de3699,0,d2aecfd8) at sys_ioctl+0x16
syscall2(d743ed40) at syscall2+0x1ef
Xint0x80_syscall() at Xint0x80_syscall+0x36
(kgdb) backtrace
#0 dumpsys () at ./machine/thread.h:83
#1 0xc0332666 in boot (howto=260) at /usr/src/sys/kern/kern_shutdown.c:375
#2 0xc0332787 in panic (fmt=0xc059e8fc "from debugger")
at /usr/src/sys/kern/kern_shutdown.c:801
#3 0xc0182745 in db_panic (addr=-1068281776, have_addr=0, count=-1,
modif=0xd743e5c4 "") at /usr/src/sys/ddb/db_command.c:447
#4 0xc0182db0 in db_command_loop () at /usr/src/sys/ddb/db_command.c:343
#5 0xc0185380 in db_trap (type=3, code=0) at /usr/src/sys/ddb/db_trap.c:71
#6 0xc05351bc in kdb_trap (type=3, code=0, regs=0xd743e6c0)
at /usr/src/sys/platform/pc32/i386/db_interface.c:152
#7 0xc0543693 in trap (frame=0xd743e6c0)
at /usr/src/sys/platform/pc32/i386/trap.c:797
#8 0xc0535ef7 in calltrap ()
at /usr/src/sys/platform/pc32/i386/exception.s:785
#9 0xc0535050 in Debugger (msg=0xc05b6499 "panic") at ./cpu/cpufunc.h:73
#10 0xc033277e in panic (fmt=0xc059aed8 "assertion: %s in %s")
at /usr/src/sys/kern/kern_shutdown.c:799
#11 0xc04a1dc4 in hammer_recover_cursor (cursor=0xd743e7b8)
at /usr/src/sys/vfs/hammer/hammer_cursor.c:582
#12 0xc04ab39d in hammer_ioc_mirror_write (trans=0xd743ea84, ip=0xd2c52550,
mirror=0xc2a72998) at /usr/src/sys/vfs/hammer/hammer_mirror.c:457
#13 0xc04aa686 in hammer_ioctl (ip=0xd2c52550, com=3234097160,
data=0xc2a72998 "", fflag=1, cred=0xc29a7768)
at /usr/src/sys/vfs/hammer/hammer_ioctl.c:134
#14 0xc04ba16d in hammer_vop_ioctl (ap=0xd743eae0)
at /usr/src/sys/vfs/hammer/hammer_vnops.c:2191
#15 0xc0387c4e in vop_ioctl (ops=0xc2ae8b50, vp=0xc2b13fa8,
command=3234097160, data=0xc2a72998 "", fflag=1, cred=0xc29a7768)
at /usr/src/sys/kern/vfs_vopops.c:372
#16 0xc0387018 in vn_ioctl (fp=0xd0291148, com=3234097160,
data=0xc2a72998 "",
ucred=0xc29a7768) at /usr/src/sys/kern/vfs_vnops.c:1120
#17 0xc035295b in mapped_ioctl (fd=3, com=3234097160,
uspc_data=0xbfbff794 <Address 0xbfbff794 out of bounds>, map=0x0)
at /usr/src/sys/sys/file2.h:87
#18 0xc03529e3 in sys_ioctl (uap=0xd743ecf0)
at /usr/src/sys/kern/sys_generic.c:525
#19 0xc0543033 in syscall2 (frame=0xd743ed40)
at /usr/src/sys/platform/pc32/i386/trap.c:1339
#20 0xc0535fa6 in Xint0x80_syscall ()
at /usr/src/sys/platform/pc32/i386/exception.s:876
#21 0x080552d7 in ?? ()
Backtrace stopped: previous frame inner to this frame (corrupt stack?)
(kgdb) list *0xc04a1dc4
0xc04a1dc4 is in hammer_recover_cursor
(/usr/src/sys/vfs/hammer/hammer_cursor.c:587).
582 KKASSERT(cursor->trans->sync_lock_refs > 0);
583
584 /*
585 * Wait for the deadlock to clear
586 */
587 if (cursor->deadlk_node) {
588 hammer_lock_ex_ident(&cursor->deadlk_node->lock,
"hmrdlk");
589 hammer_unlock(&cursor->deadlk_node->lock);
590 hammer_rel_node(cursor->deadlk_node);
591 cursor->deadlk_node = NULL;
(kgdb) list *0xc04ab39d
0xc04ab39d is in hammer_ioc_mirror_write
(/usr/src/sys/vfs/hammer/hammer_mirror.c:458).
453 * for the next loop.
454 */
455 if (error == EDEADLK) {
456 while (error == EDEADLK) {
457 hammer_recover_cursor(&cursor);
458 error =
hammer_cursor_upgrade(&cursor);
459 }
460 } else {
461 if (error == EALREADY)
462 error = 0;
(kgdb) list *0xc04aa686
0xc04aa686 is in hammer_ioctl (/usr/src/sys/vfs/hammer/hammer_ioctl.c:134).
129 (struct hammer_ioc_mirror_rw
*)data);
130 }
131 break;
132 case HAMMERIOC_MIRROR_WRITE:
133 if (error == 0) {
134 error = hammer_ioc_mirror_write(&trans, ip,
135 (struct hammer_ioc_mirror_rw
*)data);
136 }
137 break;
138 case HAMMERIOC_GET_VERSION:
(kgdb) list *0xc04ba16d
0xc04ba16d is in hammer_vop_ioctl
(/usr/src/sys/vfs/hammer/hammer_vnops.c:2193).
2188 struct hammer_inode *ip = ap->a_vp->v_data;
2189
2190 ++hammer_stats_file_iopsr;
2191 return(hammer_ioctl(ip, ap->a_command, ap->a_data,
2192 ap->a_fflag, ap->a_cred));
2193 }
2194
2195 static
2196 int
2197 hammer_vop_mountctl(struct vop_mountctl_args *ap)
(kgdb) list *0xc0387c4e
0xc0387c4e is in vop_ioctl (/usr/src/sys/kern/vfs_vopops.c:374).
369 ap.a_fflag = fflag;
370 ap.a_cred = cred;
371
372 DO_OPS(ops, error, &ap, vop_ioctl);
373 return(error);
374 }
375
376 int
377 vop_poll(struct vop_ops *ops, struct vnode *vp, int events,
struct ucred *cred)
378 {
(kgdb) list *0xc0387018
0xc0387018 is in vn_ioctl (/usr/src/sys/kern/vfs_vnops.c:1120).
1115 }
1116 *(int *)data = dev_dflags(vp->v_rdev) &
D_TYPEMASK;
1117 error = 0;
1118 break;
1119 }
1120 error = VOP_IOCTL(vp, com, data, fp->f_flag, ucred);
1121 if (error == 0 && com == TIOCSCTTY) {
1122 struct proc *p = curthread->td_proc;
1123 struct session *sess;
1124
(kgdb)
> Best update to latest master, there were many fixes since August.
Now I'm planning to start updating the system and watch if crashes will
go on. Please tell, what can I dig from crash dumps more, or how to send
those dumps to You?