possible deadlock with InterprocessSyncFile
UTs stopped running at PgSqlLeaseMgrDbLostCallbackTest.testDbLostCallback
while running unit tests for all modules with GTEST_SHUFFLE=1
. Looks like a deadlock involving InterprocessSyncFile
.
(gdb) thread apply all bt
Thread 5 (Thread 0x7ffad3d01640 (LWP 3207223) "lt-libdhcpsrv_u"):
#0 0x00007ffad681d6a2 in pthread_cond_wait@@GLIBC_2.3.2 () from /usr/lib/libpthread.so.0
#1 0x00007ffad6041c11 in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /build/gcc/src/gcc-build/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865
#2 std::condition_variable::wait (this=<optimized out>, __lock=...) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53
#3 0x00007ffad67cf8e2 in ?? () from /usr/lib/liblog4cplus-2.0.so.3
#4 0x00007ffad6047c24 in std::execute_native_thread_routine (__p=0x562068e7cd00) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/thread.cc:80
#5 0x00007ffad68173e9 in start_thread () from /usr/lib/libpthread.so.0
#6 0x00007ffad5d4f293 in clone () from /usr/lib/libc.so.6
Thread 4 (Thread 0x7ffad4502640 (LWP 3207222) "lt-libdhcpsrv_u"):
#0 0x00007ffad681d6a2 in pthread_cond_wait@@GLIBC_2.3.2 () from /usr/lib/libpthread.so.0
#1 0x00007ffad6041c11 in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /build/gcc/src/gcc-build/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865
#2 std::condition_variable::wait (this=<optimized out>, __lock=...) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53
#3 0x00007ffad67cf8e2 in ?? () from /usr/lib/liblog4cplus-2.0.so.3
#4 0x00007ffad6047c24 in std::execute_native_thread_routine (__p=0x562068e7cde0) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/thread.cc:80
#5 0x00007ffad68173e9 in start_thread () from /usr/lib/libpthread.so.0
#6 0x00007ffad5d4f293 in clone () from /usr/lib/libc.so.6
Thread 3 (Thread 0x7ffad4d03640 (LWP 3207221) "lt-libdhcpsrv_u"):
#0 0x00007ffad681d6a2 in pthread_cond_wait@@GLIBC_2.3.2 () from /usr/lib/libpthread.so.0
#1 0x00007ffad6041c11 in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /build/gcc/src/gcc-build/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865
#2 std::condition_variable::wait (this=<optimized out>, __lock=...) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53
#3 0x00007ffad67cf8e2 in ?? () from /usr/lib/liblog4cplus-2.0.so.3
#4 0x00007ffad6047c24 in std::execute_native_thread_routine (__p=0x562068e7c680) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/thread.cc:80
#5 0x00007ffad68173e9 in start_thread () from /usr/lib/libpthread.so.0
#6 0x00007ffad5d4f293 in clone () from /usr/lib/libc.so.6
Thread 2 (Thread 0x7ffad5504640 (LWP 3207220) "lt-libdhcpsrv_u"):
#0 0x00007ffad681d6a2 in pthread_cond_wait@@GLIBC_2.3.2 () from /usr/lib/libpthread.so.0
#1 0x00007ffad6041c11 in __gthread_cond_wait (__mutex=<optimized out>, __cond=<optimized out>) at /build/gcc/src/gcc-build/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:865
#2 std::condition_variable::wait (this=<optimized out>, __lock=...) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/condition_variable.cc:53
#3 0x00007ffad67cf8e2 in ?? () from /usr/lib/liblog4cplus-2.0.so.3
#4 0x00007ffad6047c24 in std::execute_native_thread_routine (__p=0x562068e7cce0) at /build/gcc/src/gcc/libstdc++-v3/src/c++11/thread.cc:80
#5 0x00007ffad68173e9 in start_thread () from /usr/lib/libpthread.so.0
#6 0x00007ffad5d4f293 in clone () from /usr/lib/libc.so.6
Thread 1 (Thread 0x7ffad5510040 (LWP 3207218) "lt-libdhcpsrv_u"):
#0 0x00007ffad5d40427 in fcntl64 () from /usr/lib/libc.so.6
#1 0x00007ffad6820eff in fcntl_compat () from /usr/lib/libpthread.so.0
#2 0x00007ffad72321c3 in isc::log::interprocess::InterprocessSyncFile::do_lock (this=0x562068fee310, cmd=7, l_type=1) at interprocess_sync_file.cc:85
#3 0x00007ffad7232405 in isc::log::interprocess::InterprocessSyncFile::lock (this=0x562068fee310) at interprocess_sync_file.cc:94
#4 0x00007ffad720f9de in isc::log::interprocess::InterprocessSyncLocker::lock (this=0x7ffcafeaf338) at ../../../src/lib/log/interprocess/interprocess_sync.h:109
#5 0x00007ffad720ea5a in isc::log::LoggerImpl::outputRaw (this=0x562068fa5130, severity=@0x7ffcafeaf4c8: isc::log::ERROR, message="DATABASE_PGSQL_FATAL_ERROR Unrecoverable PostgreSQL error occurred: Statement: <get_lease4_addr>, reason: could not receive data from server: Bad file descriptor\n (error code: <sqlstate null>).") at logger_impl.cc:162
#6 0x00007ffad720c19c in isc::log::Logger::output (this=0x7ffad7528e60 <isc::db::database_logger>, severity=@0x7ffcafeaf4c8: isc::log::ERROR, message="DATABASE_PGSQL_FATAL_ERROR Unrecoverable PostgreSQL error occurred: Statement: <get_lease4_addr>, reason: could not receive data from server: Bad file descriptor\n (error code: <sqlstate null>).") at logger.cc:147
#7 0x000056206621ee95 in isc::log::Formatter<isc::log::Logger>::~Formatter (this=0x7ffcafeaf4c0, __in_chrg=<optimized out>) at ../../../../src/lib/log/log_formatter.h:162
#8 0x00007ffad8057414 in isc::db::PgSqlConnection::checkStatementError (this=0x562068fa4e30, r=..., statement=...) at pgsql_connection.cc:333
#9 0x00007ffad9388f4c in isc::dhcp::PgSqlLeaseMgr::getLeaseCollection<boost::scoped_ptr<isc::dhcp::PgSqlLease4Exchange>, std::vector<boost::shared_ptr<isc::dhcp::Lease4>, std::allocator<boost::shared_ptr<isc::dhcp::Lease4> > > > (this=0x562068fb2430, ctx=..., stindex=isc::dhcp::PgSqlLeaseMgr::GET_LEASE4_ADDR, bind_array=..., exchange=..., result=std::vector of length 0, capacity 0, single=true) at pgsql_lease_mgr.cc:1354
#10 0x00007ffad9378971 in isc::dhcp::PgSqlLeaseMgr::getLease (this=0x562068fb2430, ctx=..., stindex=isc::dhcp::PgSqlLeaseMgr::GET_LEASE4_ADDR, bind_array=..., result=...) at pgsql_lease_mgr.cc:1378
#11 0x00007ffad9378de9 in isc::dhcp::PgSqlLeaseMgr::getLease4 (this=0x562068fb2430, addr=...) at pgsql_lease_mgr.cc:1429
#12 0x00005620667d255b in isc::dhcp::test::LeaseMgrDbLostCallbackTest::testDbLostCallback (this=0x562068fb3a40) at generic_lease_mgr_unittest.cc:3318
#13 0x000056206692e5df in (anonymous namespace)::PgSqlLeaseMgrDbLostCallbackTest_testDbLostCallback_Test::TestBody (this=0x562068fb3a40) at pgsql_lease_mgr_unittest.cc:942
#14 0x00007ffad61ab807 in void testing::internal::HandleExceptionsInMethodIfSupported<testing::Test, void>(testing::Test*, void (testing::Test::*)(), char const*) () from /usr/lib/libgtest.so.1.10.0
#15 0x00007ffad619f091 in testing::Test::Run() () from /usr/lib/libgtest.so.1.10.0
#16 0x00007ffad619f1ef in testing::TestInfo::Run() () from /usr/lib/libgtest.so.1.10.0
#17 0x00007ffad619f2d7 in testing::TestSuite::Run() () from /usr/lib/libgtest.so.1.10.0
#18 0x00007ffad619f854 in testing::internal::UnitTestImpl::RunAllTests() () from /usr/lib/libgtest.so.1.10.0
#19 0x00007ffad61abe37 in bool testing::internal::HandleExceptionsInMethodIfSupported<testing::internal::UnitTestImpl, bool>(testing::internal::UnitTestImpl*, bool (testing::internal::UnitTestImpl::*)(), char const*) () from /usr/lib/libgtest.so.1.10.0
#20 0x00007ffad619fa7a in testing::UnitTest::Run() () from /usr/lib/libgtest.so.1.10.0
#21 0x0000562065f96f78 in RUN_ALL_TESTS () at /usr/include/gtest/gtest.h:2473
#22 0x0000562065f96e8a in main (argc=1, argv=0x7ffcafeaff88) at run_unittests.cc:17
$ lsof -p 3207218 2>&1 | grep --color=auto lockfile
lt-libdhc 3207218 andrei 6u REG 259,8 0 29758074 /home/andrei/work/isc/kea-1574-make-shell-tests-and-shell-scripts-more-robust/logger_lockfile
lt-libdhc 3207218 andrei 7u REG 259,8 0 29758074 /home/andrei/work/isc/kea-1574-make-shell-tests-and-shell-scripts-more-robust/logger_lockfile
lt-libdhc 3207218 andrei 8u REG 259,8 0 29758074 /home/andrei/work/isc/kea-1574-make-shell-tests-and-shell-scripts-more-robust/logger_lockfile
lt-libdhc 3207218 andrei 9u REG 259,8 0 29758074 /home/andrei/work/isc/kea-1574-make-shell-tests-and-shell-scripts-more-robust/logger_lockfile
lt-libdhc 3207218 andrei 13u REG 259,8 0 29758074 /home/andrei/work/isc/kea-1574-make-shell-tests-and-shell-scripts-more-robust/logger_lockfile
$ wc -l /home/andrei/work/isc/kea-1574-make-shell-tests-and-shell-scripts-more-robust/logger_lockfile
0 /home/andrei/work/isc/kea-1574-make-shell-tests-and-shell-scripts-more-robust/logger_lockfile