Skip to content

crash happens in rdataset_disassociate

Bellow is the backtrace when crash happened. It happened when the recordset in zone is updated frequently. From backtrace, it seems the rbt node is invalid so the isc_assertion_failed failed. The locknum in rbt node is 951, this is wrong so it get a wrong lock from rbtdb->node_locks array. When crash happens, even thread 4 is destroy another rbt_tree but it is not match the memory with thread 11.

(gdb) info threads Id Target Id Frame 11 Thread 0x7f3a7945f880 (LWP 968) 0x00007f3a76296792 in sigsuspend () from /lib64/libc.so.6 10 Thread 0x7f3a74d4f700 (LWP 969) 0x00007f3a773be965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 9 Thread 0x7f3a7454e700 (LWP 970) 0x00007f3a773bcd57 in pthread_mutex_lock () from /lib64/libpthread.so.0 8 Thread 0x7f3a7354c700 (LWP 972) 0x00007f3a773be965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 7 Thread 0x7f3a70d47700 (LWP 977) 0x00007f3a773bed12 in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 6 Thread 0x7f3a70546700 (LWP 978) 0x00007f3a7635a0b3 in epoll_wait () from /lib64/libc.so.6 5 Thread 0x7f3a73d4d700 (LWP 971) 0x00007f3a773be965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 4 Thread 0x7f3a71548700 (LWP 976) 0x00007f3a773be965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 3 Thread 0x7f3a71d49700 (LWP 975) 0x00007f3a773be965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0 2 Thread 0x7f3a7254a700 (LWP 974) 0x00007f3a773be965 in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0

  • 1 Thread 0x7f3a72d4b700 (LWP 973) 0x00007f3a76296417 in raise () from /lib64/libc.so.6 (gdb) f 11 #11 (closed) 0x000000000042761a in ns_client_endrequest (client=0x7f3a68198660) at client.c:881 881 client.c: No such file or directory. (gdb) bt #0 0x00007f3a76296417 in raise () from /lib64/libc.so.6 #1 0x00007f3a76297b08 in abort () from /lib64/libc.so.6 #2 0x000000000043a046 in assertion_failed (file=, line=, type=, cond=) at ./main.c:250 #3 0x00007f3a77a47d9a in isc_assertion_failed (file=file@entry=0x7f3a77a8c81c "rwlock.c", line=line@entry=245, type=type@entry=isc_assertiontype_require, cond=cond@entry=0x7f3a77a8c8f0 "(__builtin_expect(!!((rwl) != ((void *)0)), 1) && __builtin_expect(!!(((const isc__magic_t *)(rwl))->magic == ((('R') << 24 | ('W') << 16 | ('L') << 8 | ('k')))), 1))") at assertions.c:58 #4 (closed) 0x00007f3a77a633e3 in isc_rwlock_lock (rwl=rwl@entry=0x7f3704188978, type=type@entry=isc_rwlocktype_read) at rwlock.c:245 #5 (closed) 0x00007f3a78cf0a08 in detachnode (db=0x7f371bab95a0, targetp=targetp@entry=0x7f3a72d492e8) at rbtdb.c:5425 #6 (closed) 0x00007f3a78cf0c2e in rdataset_disassociate (rdataset=) at rbtdb.c:8466 #7 (closed) 0x00007f3a78d47ce8 in dns_rdataset_disassociate (rdataset=rdataset@entry=0x7f3a6f8feef0) at rdataset.c:123 #8 (closed) 0x00007f3a78cc4c81 in msgresetnames (first_section=0, msg=0x7f3a6f8f5010, msg@entry=0x7f3a6f8fb060) at message.c:451 #9 (closed) msgreset (msg=msg@entry=0x7f3a6f8f5010, everything=everything@entry=isc_boolean_false) at message.c:535 #10 (closed) 0x00007f3a78cc5a7a in dns_message_reset (msg=0x7f3a6f8f5010, intent=intent@entry=1) at message.c:792 #11 (closed) 0x000000000042761a in ns_client_endrequest (client=0x7f3a68198660) at client.c:881 #12 (closed) exit_check (client=0x7f3a68198660) at client.c:534 #13 (closed) 0x00000000004292d0 in ns_client_detach (clientp=clientp@entry=0x7f3a72d494e8) at client.c:3386 #14 (closed) 0x0000000000446ef8 in query_find (client=0x0, event=event@entry=0x0, qtype=qtype@entry=1) at query.c:8417 #15 (closed) 0x0000000000450ea5 in ns_query_start (client=client@entry=0x7f3a68198660) at query.c:8708 #16 (closed) 0x000000000042c2d2 in client_request (task=, event=) at client.c:2827 #17 (closed) 0x00007f3a77a6b4f3 in dispatch (manager=0x7f3a79422010) at task.c:1128 #18 (closed) run (uap=0x7f3a79422010) at task.c:1300 #19 (closed) 0x00007f3a773bae45 in start_thread () from /lib64/libpthread.so.0 #20 (closed) 0x00007f3a76359add in clone () from /lib64/libc.so.6 (gdb) f 4 #4 (closed) 0x00007f3a77a633e3 in isc_rwlock_lock (rwl=rwl@entry=0x7f3704188978, type=type@entry=isc_rwlocktype_read) at rwlock.c:245 245 rwlock.c: No such file or directory. (gdb) info local No locals. (gdb) p entry No symbol "entry" in current context. (gdb) p rwl $6 = (isc_rwlock_t *) 0x7f3704188978 (gdb) p *rwl $7 = {magic = 3200171710, lock = {__data = {__lock = -1094795586, __count = 3200171710, __owner = -1094795586, __nusers = 3200171710, __kind = -1094795586, __spins = -16706, __elision = -16706, __list = {__prev = 0xbebebebebebebebe, __next = 0xbebebebebebebebe}}, __size = '\276' <repeats 40 times>, __align = -4702111234474983746}, write_requests = -1094795586, write_completions = -1094795586, cnt_and_flag = -1094795586, readable = {__data = { __lock = -1094795586, __futex = 3200171710, __total_seq = 13744632839234567870, __wakeup_seq = 13744632839234567870, __woken_seq = 13744632839234567870, __mutex = 0xbebebebebebebebe, __nwaiters = 3200171710, __broadcast_seq = 3200171710}, __size = '\276' <repeats 48 times>, __align = -4702111234474983746}, writeable = {__data = {__lock = -1094795586, __futex = 3200171710, __total_seq = 13744632839234567870, __wakeup_seq = 13744632839234567870, __woken_seq = 13744632839234567870, __mutex = 0xbebebebebebebebe, __nwaiters = 3200171710, __broadcast_seq = 3200171710}, __size = '\276' <repeats 48 times>, __align = -4702111234474983746}, readers_waiting = 3200171710, write_granted = 3200171710, write_quota = 3200171710} (gdb) p rwl->magic $8 = 3200171710 gdb) f 5 #5 (closed) 0x00007f3a78cf0a08 in detachnode (db=0x7f371bab95a0, targetp=targetp@entry=0x7f3a72d492e8) at rbtdb.c:5425 5425 rbtdb.c: No such file or directory. (gdb) info local rbtdb = 0x7f371bab95a0 node = 0x7f3719c3f5b0 want_free = isc_boolean_false inactive = isc_boolean_false nodelock = 0x7f3704188978 (gdb) p node->locknum $11 = 951 (gdb) p *node $25 = {magic = 740377912, is_root = 1, color = 1, find_callback = 1, attributes = 6, nsec = 0, namelen = 127, offsetlen = 0, oldnamelen = 0, is_mmapped = 0, parent_is_relative = 1, left_is_relative = 1, right_is_relative = 1, down_is_relative = 1, data_is_relative = 0, rpz = 1, hashval = 3739147998, uppernode = 0xdededededededede, hashnext = 0xdededededededede, parent = 0xdededededededede, left = 0xdededededededede, right = 0xdededededededede, down = 0xdededededededede, deadlink = {prev = 0xdededededededede, next = 0xdededededededede}, data = 0xdededededededede, dirty = 0, wild = 1, locknum = 951, references = {refs = -555819298}}

(gdb) t 9 [Switching to thread 9 (Thread 0x7f3a7454e700 (LWP 970))] #0 0x00007f3a773bcd57 in pthread_mutex_lock () from /lib64/libpthread.so.0 (gdb) info local No symbol table info available. (gdb) bt #0 0x00007f3a773bcd57 in pthread_mutex_lock () from /lib64/libpthread.so.0 #1 0x00007f3a77a63864 in isc_rwlock_unlock (rwl=0x7f370415e018, type=type@entry=isc_rwlocktype_write) at rwlock.c:510 #2 0x00007f3a78ce819d in delete_callback (data=, arg=0x7f371bab95a0) at rbtdb.c:7717 #3 0x00007f3a78cda981 in deletetreeflat (rbt=rbt@entry=0x7f35972fe0f8, quantum=213, unhash=unhash@entry=isc_boolean_false, nodep=nodep@entry=0x7f35972fe108) at rbt.c:2841 #4 (closed) 0x00007f3a78cdb9f8 in dns_rbt_destroy2 (rbtp=rbtp@entry=0x7f371bab9830, quantum=) at rbt.c:1022 #5 (closed) 0x00007f3a78ce34c2 in free_rbtdb (rbtdb=0x7f371bab95a0, log=isc_boolean_true, event=0x7f3895445dd0) at rbtdb.c:1185 #6 (closed) 0x00007f3a77a6b4f3 in dispatch (manager=0x7f3a79422010) at task.c:1128 #7 (closed) run (uap=0x7f3a79422010) at task.c:1300 #8 (closed) 0x00007f3a773bae45 in start_thread () from /lib64/libpthread.so.0 #9 (closed) 0x00007f3a76359add in clone () from /lib64/libc.so.6

To upload designs, you'll need to enable LFS and have an admin enable hashed storage. More information