named may crash upon shutdown when catalog zones are used (mutex destruction)
See https://gitlab.isc.org/isc-projects/bind9/-/jobs/230738. Mutex destruction fails, causing a RUNTIME_CHECK()
assertion failure. There is another thread ("Thread 9" below) in the core which is also "in the middle of" failing an assertion failure related to reference counting for a view structure. Looks like the catalog zone shutdown code handles certain event scheduling scenarios less than ideally - this crash might or might not be related to #994 (closed).
(gdb) thread apply all bt
Thread 11 (Thread 0x7f0c9c6d8700 (LWP 28074)):
#0 0x00007f0ca253c243 in epoll_wait () from /lib64/libc.so.6
#1 0x00007f0ca3e4ee4c in watcher (uap=0x7f0ca5116020) at socket.c:4299
#2 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#3 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 10 (Thread 0x7f0c9ced9700 (LWP 28073)):
#0 0x00007f0ca2fdfa5e in pthread_cond_timedwait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f0ca3e5158e in isc_condition_waituntil (c=0x7f0ca5114088, m=0x7f0ca5114038, t=0x7f0ca511407c) at condition.c:59
#2 0x00007f0ca3e3ab5c in run (uap=0x7f0ca5114020) at timer.c:806
#3 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 9 (Thread 0x7f0ca06e0700 (LWP 28066)):
#0 0x00007f0c99b4d8eb in ?? () from /lib64/libgcc_s.so.1
#1 0x00007f0ca25799d6 in dl_iterate_phdr () from /lib64/libc.so.6
#2 0x00007f0c99b4e207 in _Unwind_Find_FDE () from /lib64/libgcc_s.so.1
#3 0x00007f0c99b4b603 in ?? () from /lib64/libgcc_s.so.1
#4 0x00007f0c99b4c119 in _Unwind_Backtrace () from /lib64/libgcc_s.so.1
#5 0x00007f0ca2551c46 in backtrace () from /lib64/libc.so.6
#6 0x00007f0ca3e0bb1a in isc_backtrace_gettrace (addrs=0x7f0ca06de3d0, maxaddrs=<value optimized out>, nframes=0x7f0ca06de7ec) at backtrace.c:75
#7 0x00000000004321df in assertion_failed (file=0x7f0ca4af370c "view.c", line=583, type=isc_assertiontype_require, cond=0x7f0ca4ac4cdd "prev > 0") at ./main.c:190
#8 0x00007f0ca3e0b93a in isc_assertion_failed (file=<value optimized out>, line=<value optimized out>, type=<value optimized out>, cond=<value optimized out>) at assertions.c:51
#9 0x00007f0ca4a62eb3 in dns_view_attach (source=<value optimized out>, targetp=<value optimized out>) at view.c:583
#10 0x00000000004569b1 in catz_create_chg_task (entry=0x7f0ca5114e80, origin=0x7f0ca50a9020, view=0x7f0c7c04ab90, taskmgr=0x7f0ca5112020, udata=0x6cb370, type=262200) at ./server.c:2412
#11 0x00007f0ca4948526 in dns_catz_zones_merge (target=0x7f0ca50a9020, newzone=<value optimized out>) at catz.c:515
#12 0x00007f0ca494a05b in dns_catz_update_from_db (db=<value optimized out>, catzs=0x7f0ca50a8fa0) at catz.c:1940
#13 0x00007f0ca494a1f8 in dns_catz_update_taskaction (task=<value optimized out>, event=0x7f0ca50a9168) at catz.c:1694
#14 0x00007f0ca3e35cdc in dispatch (uap=0x7f0ca5112020) at task.c:1143
#15 run (uap=0x7f0ca5112020) at task.c:1315
#16 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#17 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 8 (Thread 0x7f0c9eedd700 (LWP 28069)):
#0 0x00007f0ca2fdf68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f0ca3e35ac6 in dispatch (uap=0x7f0ca5112020) at task.c:1089
#2 run (uap=0x7f0ca5112020) at task.c:1315
#3 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 7 (Thread 0x7f0c9e6dc700 (LWP 28070)):
#0 0x00007f0ca2fe2334 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f0ca2fdd60e in _L_lock_995 () from /lib64/libpthread.so.0
#2 0x00007f0ca2fdd576 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007f0ca4952d22 in destroy_disp (task=<value optimized out>, event=0x7f0ca50a06b0) at dispatch.c:527
#4 0x00007f0ca3e35cdc in dispatch (uap=0x7f0ca5112020) at task.c:1143
#5 run (uap=0x7f0ca5112020) at task.c:1315
#6 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 6 (Thread 0x7f0c9dedb700 (LWP 28071)):
#0 0x00007f0ca2fe2334 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f0ca2fdd60e in _L_lock_995 () from /lib64/libpthread.so.0
#2 0x00007f0ca2fdd576 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007f0ca3e1ad40 in isc_log_doit (lctx=0x1656820, category=0x6c30d0, module=0x6c3170, level=3, write_once=false, msgcat=0x0, msgset=0, msg=0, format=0x49f320 "client @%p %s%s%s%s%s%s%s%s: %s", args=0x7f0c9ded9280) at log.c:1440
#4 0x00007f0ca3e1be3c in isc_log_write (lctx=<value optimized out>, category=<value optimized out>, module=<value optimized out>, level=<value optimized out>, format=<value optimized out>) at log.c:833
#5 0x000000000041ef5e in ns_client_logv (client=0x7f0c7c994d00, category=0x6c30d0, module=0x6c3170, level=3, fmt=<value optimized out>, ap=<value optimized out>) at client.c:4049
#6 0x000000000041f105 in ns_client_log (client=0x7f0c7c994d00, category=0x6c30d0, module=0x6c3170, level=3, fmt=0x4b1d22 "%s") at client.c:4065
#7 0x00000000004237e9 in client_shutdown (task=<value optimized out>, event=0x7f0c7c600df8) at client.c:826
#8 0x00007f0ca3e35cdc in dispatch (uap=0x7f0ca5112020) at task.c:1143
#9 run (uap=0x7f0ca5112020) at task.c:1315
#10 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#11 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 5 (Thread 0x7f0c9d6da700 (LWP 28072)):
#0 0x00007f0ca2fdf68c in pthread_cond_wait@@GLIBC_2.3.2 () from /lib64/libpthread.so.0
#1 0x00007f0ca3e35ac6 in dispatch (uap=0x7f0ca5112020) at task.c:1089
#2 run (uap=0x7f0ca5112020) at task.c:1315
#3 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#4 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 4 (Thread 0x7f0c9f6de700 (LWP 28068)):
#0 0x00007f0ca2fe2334 in __lll_lock_wait () from /lib64/libpthread.so.0
#1 0x00007f0ca2fdd60e in _L_lock_995 () from /lib64/libpthread.so.0
#2 0x00007f0ca2fdd576 in pthread_mutex_lock () from /lib64/libpthread.so.0
#3 0x00007f0ca4952d22 in destroy_disp (task=<value optimized out>, event=0x7f0ca50a04d0) at dispatch.c:527
#4 0x00007f0ca3e35cdc in dispatch (uap=0x7f0ca5112020) at task.c:1143
#5 run (uap=0x7f0ca5112020) at task.c:1315
#6 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#7 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 3 (Thread 0x7f0c9fedf700 (LWP 28067)):
#0 0x00007f0ca253861a in mmap64 () from /lib64/libc.so.6
#1 0x00007f0ca24b91cc in _IO_file_doallocate_internal () from /lib64/libc.so.6
#2 0x00007f0ca24c668c in _IO_doallocbuf_internal () from /lib64/libc.so.6
#3 0x00007f0ca24c4fbc in _IO_new_file_seekoff () from /lib64/libc.so.6
#4 0x00007f0ca24c3d42 in _IO_new_file_attach () from /lib64/libc.so.6
#5 0x00007f0ca24b9685 in fdopen@@GLIBC_2.2.5 () from /lib64/libc.so.6
#6 0x00007f0ca3e411a5 in isc_file_openuniquemode (templet=0x7f0c9feddd20 "tmp-ARUQf2zBLO", mode=384, fp=0x7f0c9feded28) at file.c:360
#7 0x00007f0ca4a679bd in destroy (view=0x7f0c7d0afa80) at view.c:376
#8 0x00007f0ca4a68e48 in adb_shutdown (task=<value optimized out>, event=0x0) at view.c:757
#9 0x00007f0ca3e35cdc in dispatch (uap=0x7f0ca5112020) at task.c:1143
#10 run (uap=0x7f0ca5112020) at task.c:1315
#11 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#12 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6
Thread 2 (Thread 0x7f0ca514e7c0 (LWP 28051)):
#0 0x00007f0ca2fdc2fd in pthread_join () from /lib64/libpthread.so.0
#1 0x00007f0ca3e37b24 in isc__taskmgr_destroy (managerp=0x6caa10) at task.c:1585
#2 0x00007f0ca3e37dad in isc_taskmgr_destroy (managerp=0x6caa10) at task.c:2102
#3 0x0000000000433ead in destroy_managers (argc=<value optimized out>, argv=<value optimized out>) at ./main.c:872
#4 cleanup (argc=<value optimized out>, argv=<value optimized out>) at ./main.c:1275
#5 main (argc=<value optimized out>, argv=<value optimized out>) at ./main.c:1508
Thread 1 (Thread 0x7f0ca0ee1700 (LWP 28065)):
#0 0x00007f0ca24854f5 in raise () from /lib64/libc.so.6
#1 0x00007f0ca2486cd5 in abort () from /lib64/libc.so.6
#2 0x000000000043212b in library_fatal_error (file=0x7f0ca4acb84c "catz.c", line=<value optimized out>, format=0x7f0ca3e53020 "RUNTIME_CHECK(%s) %s", args=0x7f0ca0ee0bf0) at ./main.c:275
#3 0x00007f0ca3e0fa54 in isc_error_fatal (file=<value optimized out>, line=<value optimized out>, format=<value optimized out>) at error.c:68
#4 0x00007f0ca3e0fab4 in isc_error_runtimecheck (file=0x7f0ca4acb84c "catz.c", line=866, expression=0x7f0ca4acb0e8 "((pthread_mutex_destroy(((&catzs->lock))) == 0) ? 0 : 34) == 0") at error.c:75
#5 0x00007f0ca4947e12 in dns_catz_catzs_detach (catzsp=<value optimized out>) at catz.c:866
#6 0x00007f0ca4a82ae5 in zone_free (zone=0x7f0c944f0260) at zone.c:1223
#7 0x00007f0ca4a8b180 in zone_shutdown (task=<value optimized out>, event=<value optimized out>) at zone.c:13024
#8 0x00007f0ca3e35cdc in dispatch (uap=0x7f0ca5112020) at task.c:1143
#9 run (uap=0x7f0ca5112020) at task.c:1315
#10 0x00007f0ca2fdbaa1 in start_thread () from /lib64/libpthread.so.0
#11 0x00007f0ca253bc4d in clone () from /lib64/libc.so.6