Prevent crash on dst initialization failure

Petr Menšík requested to merge pemensik/bind9:dstlib-failure-abort into main

server might be created, but not yet fully initialized, when fatal function is called. Check both server and task before attaching exclusive task.

We detected this issue on bind-pkcs11 build with native pkcs11 build, which does not initialize softhsm tokens. Failure is expected, but crash on abort is not.

BIND 9.16.5-RedHat-9.16.5-1.fc32 (Stable Release) <id:c00b458>
running on Linux x86_64 5.7.7-200.fc32.x86_64 #1 SMP Wed Jul 1 19:53:01 UTC 2020
built by make with '--build=x86_64-redhat-linux-gnu' '--host=x86_64-redhat-linux-gnu' '--program-prefix=' '--disable-dependency-tracking' '--prefix=/usr' '--exec-prefix=/usr' '--bindir=/usr/bin' '--sbindir=/usr/sbin' '--sysconfdir=/etc' '--datadir=/usr/share' '--includedir=/usr/include' '--libdir=/usr/lib64' '--libexecdir=/usr/libexec' '--sharedstatedir=/var/lib' '--mandir=/usr/share/man' '--infodir=/usr/share/info' '--with-python=/usr/bin/python3' '--with-libtool' '--localstatedir=/var' '--with-pic' '--disable-static' '--includedir=/usr/include/bind9' '--with-tuning=large' '--with-libidn2' '--with-maxminddb' '--enable-native-pkcs11' '--with-pkcs11=/usr/lib64/pkcs11/libsofthsm2.so' '--with-dlopen=yes' '--with-dlz-ldap=yes' '--with-dlz-postgres=yes' '--with-dlz-mysql=yes' '--with-dlz-filesystem=yes' '--with-gssapi=yes' '--disable-isc-spnego' '--with-lmdb=yes' '--without-libjson' '--with-json-c' '--enable-dnstap' '--with-cmocka' '--enable-fixed-rrset' '--with-docbook-xsl=/usr/share/sgml/docbook/xsl-stylesheets' '--enable-full-report' 'build_alias=x86_64-redhat-linux-gnu' 'host_alias=x86_64-redhat-linux-gnu' 'CFLAGS= -O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fexceptions -fstack-protector-strong -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -specs=/usr/lib/rpm/redhat/redhat-annobin-cc1 -m64 -mtune=generic -fasynchronous-unwind-tables -fstack-clash-protection -fcf-protection' 'LDFLAGS=-Wl,-z,relro -Wl,--as-needed -Wl,-z,now -specs=/usr/lib/rpm/redhat/redhat-hardened-ld' 'LT_SYS_LIBRARY_PATH=/usr/lib64:' 'PKG_CONFIG_PATH=:/usr/lib64/pkgconfig:/usr/share/pkgconfig'
compiled by GCC 10.1.1 20200507 (Red Hat 10.1.1-1)
compiled with OpenSSL version: OpenSSL 1.1.1g FIPS  21 Apr 2020
linked to OpenSSL version: OpenSSL 1.1.1g FIPS  21 Apr 2020
compiled with libxml2 version: 2.9.10
linked to libxml2 version: 20910
compiled with json-c version: 0.13.1
linked to json-c version: 0.13.1
compiled with zlib version: 1.2.11
linked to zlib version: 1.2.11
linked to maxminddb version: 1.4.2
compiled with protobuf-c version: 1.3.2
linked to protobuf-c version: 1.3.2
threads support is enabled

default paths:
  named configuration:  /etc/named.conf
  rndc configuration:   /etc/rndc.conf
  DNSSEC root key:      /etc/bind.keys
  nsupdate session key: /var/run/named/session.key
  named PID file:       /var/run/named/named.pid
  named lock file:      /var/run/named/named.lock
  geoip-directory:      /usr/share/GeoIP


(gdb) bt full
#0  __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50
        set = {__val = {16387, 93866323553696, 0 <repeats 14 times>}}
        pid = <optimized out>
        tid = <optimized out>
        ret = <optimized out>
#1  0x00007f502dc41895 in __GI_abort () at abort.c:79
        save_stage = 1
        act = {__sigaction_handler = {sa_handler = 0x0, sa_sigaction = 0x0}, sa_mask = {__val = {7, 0, 0, 93866350878344, 140732340626880, 
              140732340626860, 139982349275478, 1637, 139982349268976, 140732340626784, 139982342893158, 140732340626880, 140732340628504, 549755813898, 
              5577438938329795328, 140732340626880}}, sa_flags = -186665812, sa_restorer = 0x665}
        sigs = {__val = {32, 0 <repeats 15 times>}}
#2  0x0000555ef4df13fa in assertion_failed (file=<optimized out>, line=-852761168, type=<optimized out>, cond=0x7ffecd2be5b8 "\230\266\r*P\177")
    at ../../../bin/named-pkcs11/main.c:260
        tracebuf = {0x555ef4dfb441 <assertion_failed+97>, 0x7f502e2f50a0 <isc_assertion_failed+16>, 0x7f502e31eeda <isc_task_beginexclusive+426>, 
          0x555ef4e00677 <fatal+23>, 0x555ef4e06595 <named_server_create+1333>, 0x555ef4df21ef <main+3567>, 0x7f502dc43042 <__libc_start_main+242>, 
          0x555ef4df2dce <_start+46>, 0x555ef4df2dce <_start+46>, 0x0, 0x3, 0x0, 0x0, 0xb2f2b0b9, 0x555ef6801e88, 0x7f502e341f12, 0x7ffecd2be798, 
          0x7ffecd2be6d0, 0x7ffecd2be6e0, 0x7f502e6433b1 <_dl_lookup_symbol_x+289>, 0x0, 0x0, 0x2, 0x0, 0x0, 0x555ef6801af0, 0x7ffecd2be780, 0x0, 
          0x555ef6801af0, 0xf3944b00, 0x555ef6801e88, 0x0, 0x7ffecd2be760, 0x4d670b29f3944b00, 0xffffffff, 0x555ef6819850, 0x7f502a0ddd80, 
          0x555ef6801af0, 0x555ef68198f0, 0x7f502a147632 <MutexLocker::~MutexLocker()+50>, 0x555ef6819850, 
          0x7f502a16b9cc <ObjectStore::ObjectStore(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> >)+668>, 
          0x555ef6819858, 0x555ef6819870, 0x0, 0x0, 0x0, 0x7f502e341f12, 0x7f502e323f31 <pkcs_C_Finalize+33>, 0x7ffecd2beae8, 
          0x555ef4e5b198 <named_g_mctx>, 0x7f502dd58aa4 <do_sym+116>, 0x2, 0x0, 0x0, 0x4d670b29f3944b00, 0x7ffecd2be840, 0x555ef6819770, 0x7ffecd2be7c8, 
          0x7f502a0ddd80, 0x7ffecd2be840, 0x7ffecd2be870, 0x7ffecd2be850, 0x7f502a181c1d <SessionObjectStore::~SessionObjectStore()+269>, 
          0x7ffecd2be840, 0x7ffe00000000, 0x0, 0x7ffecd2be7c8, 0x7ffecd2be7c8, 0x4d670b29f3944b00, 0x555ef6819770, 0x4d670b29f3944b00, 0x7ffecd2be880, 
          0x7ffecd2be9d0, 0x555ef68023f8, 0x0, 0x0, 0x7ffecd2beae8, 0x555ef4e5b198 <named_g_mctx>, 0x7f502de0a508 <dlsym_doit+24>, 0x555ef68023e8, 
          0x7f502dd591c8 <__GI__dl_catch_exception+136>, 0x7ffecd2be970, 0x0, 0x7f502de0a4f0 <dlsym_doit>, 0x7ffecd2be9d0, 0x18, 0x7f502e354000, 
          0x7ffecd2be970, 0x7ffecd2be87c, 0x555ef68023e8, 0xf64ffef6d3f69729, 0xffffffffffffff00, 0x0, 0x7ffecd2beae8, 0x555ef4e5b198 <named_g_mctx>, 
          0xa30f89a144a69729, 0xa252485db7649729, 0x0, 0x7ffecd2beae8, 0x555ef4e5b198 <named_g_mctx>, 0x7f502a0fd3e8 <C_Initialize(CK_VOID_PTR)+24>, 
          0x555ef67554a0, 0x7f502e2f2c43 <pk11_initialize+339>, 0x1c, 0x20, 0x7f502e072100 <ossl_pers_string>, 0x28, 0x7fffffff, 
          0x7f502dfc45ae <RAND_DRBG_instantiate+510>, 0x0, 0x555ef6802b60, 0x555ef68010f0, 0x4d670b29f3944b00, 0x38, 0x4d670b29f3944b00, 0x555ef68023f0, 
          0x7f502dd59293 <__GI__dl_catch_error+51>, 0x0, 0x0, 0x0, 0x4d670b29f3944b00, 0x555ef68023e0, 0x7ffecd2be9d0, 0x7f502de0a4f0 <dlsym_doit>, 
          0x7f502a128208 <SoftHSM::i()+56>, 0x7f502e664000 <_rtld_local>, 0x7f502e354780}
        i = <optimized out>
        nframes = 8
        result = <optimized out>
        logsuffix = <optimized out>
        fname = 0x0
#3  0x00007f502e2f50a0 in isc_assertion_failed () from /lib64/libisc.so.1605
No symbol table info available.
#4  0x00007f502e31eeda in isc_task_beginexclusive () from /lib64/libisc.so.1605
No symbol table info available.
#5  0x0000555ef4e00677 in fatal (server=server@entry=0x555ef677ee10, msg=msg@entry=0x555ef4e3dbf5 "initializing DST", result=458752)
    at ../../../bin/named-pkcs11/server.c:10114
No locals.
#6  0x0000555ef4e06595 in named_server_create (mctx=0x555ef67554a0, serverp=0x555ef4e5b148 <named_g_server>) at ../../../bin/named-pkcs11/server.c:9910
        result = <optimized out>
        server = 0x555ef677ee10
#7  0x0000555ef4df21ef in setup () at ../../../bin/named-pkcs11/main.c:1294
        result = <optimized out>
        old_openfiles = 140732340628200
        sctx = <optimized out>
        result = <optimized out>
        old_openfiles = <optimized out>
        sctx = <optimized out>
#8  main (argc=<optimized out>, argv=<optimized out>) at ../../../bin/named-pkcs11/main.c:1562
        result = <optimized out>

(gdb) frame 5
#5  0x0000555ef4e00677 in fatal (server=server@entry=0x555ef677ee10, msg=msg@entry=0x555ef4e3dbf5 "initializing DST", result=458752)
    at ../../../bin/named-pkcs11/server.c:10114
10114			(void)isc_task_beginexclusive(server->task);
(gdb) p *server
$1 = {magic = 4135055304, mctx = 0x555ef67554a0, sctx = 0x0, task = 0x0, statsfile = 0x0, dumpfile = 0x0, secrootsfile = 0x0, bindkeysfile = 0x0, 
  recfile = 0x0, version_set = false, version = 0x0, hostname_set = false, hostname = 0x0, loadmgr = 0x0, zonemgr = 0x0, viewlist = {head = 0x0, 
    tail = 0x0}, kasplist = {head = 0x0, tail = 0x0}, interfacemgr = 0x0, in_roothints = 0x0, interface_timer = 0x0, heartbeat_timer = 0x0, 
  pps_timer = 0x0, tat_timer = 0x0, interface_interval = 0, heartbeat_interval = 0, reload_event_lock = {__data = {__lock = 0, __count = 0, __owner = 0, 
      __nusers = 0, __kind = 0, __spins = 0, __elision = 0, __list = {__prev = 0x0, __next = 0x0}}, __size = '\000' <repeats 39 times>, __align = 0}, 
  reload_event = 0x0, reload_status = NAMED_RELOAD_DONE, flushonshutdown = false, cachelist = {head = 0x0, tail = 0x0}, zonestats = 0x0, 
  resolverstats = 0x0, sockstats = 0x0, controls = 0x0, dispatchgen = 0, dispatches = {head = 0x0, tail = 0x0}, statschannels = {head = 0x0, 
    tail = 0x0}, sessionkey = 0x0, session_keyfile = 0x0, session_keyname = 0x0, session_keyalg = 0, session_keybits = 0, interface_auto = false, 
  secret = '\000' <repeats 31 times>, cookiealg = ns_cookiealg_aes, dtenv = 0x0, lockfile = 0x0}

Both v9.16 and ~"v9.11" releases are affected. Probably also master, haven't checked that. Could be reproduced on Fedora 32, just:

dnf install -y bind-pkcs11
systemctl restart named-pkcs11
coredumpctl list

rhbz#1859454

Edited by Petr Menšík

Merge request reports