Thread sanitizer warnings with std::bind
I ran a DHCPv4 server compiled with a thread sanitizer, and I got the following sporadic failure:
==================
WARNING: ThreadSanitizer: data race (pid=95142)
Write of size 8 at 0x7b0c0000c220 by main thread:
#0 operator delete(void*, unsigned long) ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:150 (libtsan.so.0+0x8e878)
#1 std::_Function_base::_Base_manager<std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)> >::_M_destroy(std::_Any_data&, std::integral_constant<bool, false>) /usr/include/c++/11/bits/std_function.h:175 (kea-dhcp4+0xbe1ab)
#2 std::_Function_base::_Base_manager<std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)> >::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation) /usr/include/c++/11/bits/std_function.h:203 (kea-dhcp4+0xbe1ab)
#3 std::_Function_handler<void (), std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)> >::_M_manager(std::_Any_data&, std::_Any_data const&, std::_Manager_operation) /usr/include/c++/11/bits/std_function.h:282 (kea-dhcp4+0xbe1ab)
#4 std::_Function_base::~_Function_base() /usr/include/c++/11/bits/std_function.h:244 (kea-dhcp4+0xbccd2)
#5 std::function<void ()>::~function() /usr/include/c++/11/bits/std_function.h:334 (kea-dhcp4+0xbccd2)
#6 boost::detail::sp_ms_deleter<std::function<void ()> >::destroy() /usr/include/boost/smart_ptr/make_shared_object.hpp:59 (kea-dhcp4+0xbccd2)
#7 boost::detail::sp_ms_deleter<std::function<void ()> >::operator()(std::function<void ()>*) /usr/include/boost/smart_ptr/make_shared_object.hpp:93 (kea-dhcp4+0xbccd2)
#8 boost::detail::sp_counted_impl_pd<std::function<void ()>*, boost::detail::sp_ms_deleter<std::function<void ()> > >::dispose() /usr/include/boost/smart_ptr/detail/sp_counted_impl.hpp:169 (kea-dhcp4+0xbccd2)
#9 boost::detail::sp_counted_base::release() /usr/include/boost/smart_ptr/detail/sp_counted_base_gcc_atomic.hpp:120 (kea-dhcp4+0x5420a)
#10 boost::detail::shared_count::~shared_count() /usr/include/boost/smart_ptr/detail/shared_count.hpp:432 (kea-dhcp4+0xb845a)
#11 boost::shared_ptr<std::function<void ()> >::~shared_ptr() /usr/include/boost/smart_ptr/shared_ptr.hpp:335 (kea-dhcp4+0xb845a)
#12 isc::dhcp::Dhcpv4Srv::runOne() /home/marcin/devel/kea/src/bin/dhcp4/dhcp4_srv.cc:1113 (kea-dhcp4+0xb845a)
#13 isc::dhcp::Dhcpv4Srv::run() /home/marcin/devel/kea/src/bin/dhcp4/dhcp4_srv.cc:1023 (kea-dhcp4+0xb87b7)
#14 main /home/marcin/devel/kea/src/bin/dhcp4/main.cc:282 (kea-dhcp4+0x5082a)
Previous read of size 8 at 0x7b0c0000c220 by thread T6:
#0 boost::shared_ptr<isc::hooks::CalloutHandle> isc::dhcp::getCalloutHandle<boost::shared_ptr<isc::dhcp::Pkt4> >(boost::shared_ptr<isc::dhcp::Pkt4> const&) ../../../src/lib/dhcpsrv/callout_handle_store.h:50 (kea-dhcp4+0xb7f7d)
#1 isc::dhcp::Dhcpv4Srv::processPacketAndSendResponse(boost::shared_ptr<isc::dhcp::Pkt4>&) /home/marcin/devel/kea/src/bin/dhcp4/dhcp4_srv.cc:1139 (kea-dhcp4+0xb7f7d)
#2 isc::dhcp::Dhcpv4Srv::processPacketAndSendResponseNoThrow(boost::shared_ptr<isc::dhcp::Pkt4>&) /home/marcin/devel/kea/src/bin/dhcp4/dhcp4_srv.cc:1122 (kea-dhcp4+0xb88c7)
#3 void std::__invoke_impl<void, void (isc::dhcp::Dhcpv4Srv::*&)(boost::shared_ptr<isc::dhcp::Pkt4>&), isc::dhcp::Dhcpv4Srv*&, boost::shared_ptr<isc::dhcp::Pkt4>&>(std::__invoke_memfun_deref, void (isc::dhcp::Dhcpv4Srv::*&)(boost::shared_ptr<isc::dhcp::Pkt4>&), isc::dhcp::Dhcpv4Srv*&, boost::shared_ptr<isc::dhcp::Pkt4>&) /usr/include/c++/11/bits/invoke.h:74 (kea-dhcp4+0xba3c9)
#4 std::__invoke_result<void (isc::dhcp::Dhcpv4Srv::*&)(boost::shared_ptr<isc::dhcp::Pkt4>&), isc::dhcp::Dhcpv4Srv*&, boost::shared_ptr<isc::dhcp::Pkt4>&>::type std::__invoke<void (isc::dhcp::Dhcpv4Srv::*&)(boost::shared_ptr<isc::dhcp::Pkt4>&), isc::dhcp::Dhcpv4Srv*&, boost::shared_ptr<isc::dhcp::Pkt4>&>(void (isc::dhcp::Dhcpv4Srv::*&)(boost::shared_ptr<isc::dhcp::Pkt4>&), isc::dhcp::Dhcpv4Srv*&, boost::shared_ptr<isc::dhcp::Pkt4>&) /usr/include/c++/11/bits/invoke.h:96 (kea-dhcp4+0xba3c9)
#5 void std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)>::__call<void, , 0ul, 1ul>(std::tuple<>&&, std::_Index_tuple<0ul, 1ul>) /usr/include/c++/11/functional:420 (kea-dhcp4+0xba3c9)
#6 void std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)>::operator()<, void>() /usr/include/c++/11/functional:503 (kea-dhcp4+0xba3c9)
#7 void std::__invoke_impl<void, std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)>&>(std::__invoke_other, std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)>&) /usr/include/c++/11/bits/invoke.h:61 (kea-dhcp4+0xba3c9)
#8 std::enable_if<is_invocable_r_v<void, std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)>&>, void>::type std::__invoke_r<void, std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)>&>(std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)>&) /usr/include/c++/11/bits/invoke.h:111 (kea-dhcp4+0xba3c9)
#9 std::_Function_handler<void (), std::_Bind<void (isc::dhcp::Dhcpv4Srv::*(isc::dhcp::Dhcpv4Srv*, boost::shared_ptr<isc::dhcp::Pkt4>))(boost::shared_ptr<isc::dhcp::Pkt4>&)> >::_M_invoke(std::_Any_data const&) /usr/include/c++/11/bits/std_function.h:290 (kea-dhcp4+0xba3c9)
#10 std::function<void ()>::operator()() const /usr/include/c++/11/bits/std_function.h:590 (libkea-util.so.80+0x3ef83)
#11 isc::util::ThreadPool<std::function<void ()>, std::deque<boost::shared_ptr<std::function<void ()> >, std::allocator<boost::shared_ptr<std::function<void ()> > > > >::run() ../../../src/lib/util/thread_pool.h:599 (libkea-util.so.80+0x3ef83)
Thread T6 (tid=95149, running) created by main thread at:
#0 pthread_create ../../../../src/libsanitizer/tsan/tsan_interceptors_posix.cpp:969 (libtsan.so.0+0x605b8)
#1 std::thread::_M_start_thread(std::unique_ptr<std::thread::_State, std::default_delete<std::thread::_State> >, void (*)()) <null> (libstdc++.so.6+0xdc328)
#2 isc::dhcp::ControlledDhcpv4Srv::processCommand(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&, boost::shared_ptr<isc::data::Element const>) /home/marcin/devel/kea/src/bin/dhcp4/ctrl_dhcp4_srv.cc:861 (kea-dhcp4+0x666f8)
#3 isc::dhcp::ControlledDhcpv4Srv::loadConfigFile(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) /home/marcin/devel/kea/src/bin/dhcp4/ctrl_dhcp4_srv.cc:163 (kea-dhcp4+0x6732b)
#4 isc::dhcp::ControlledDhcpv4Srv::init(std::__cxx11::basic_string<char, std::char_traits<char>, std::allocator<char> > const&) /home/marcin/devel/kea/src/bin/dhcp4/ctrl_dhcp4_srv.cc:99 (kea-dhcp4+0x677fb)
#5 main /home/marcin/devel/kea/src/bin/dhcp4/main.cc:253 (kea-dhcp4+0x507e5)
SUMMARY: ThreadSanitizer: data race ../../../../src/libsanitizer/tsan/tsan_new_delete.cpp:150 in operator delete(void*, unsigned long)
==================
It looks that it may be related to using references to shared pointers in the functions passed to std::bind. Removing the reference seems to solve the problem. For example:
diff --git a/src/bin/dhcp4/dhcp4_srv.cc b/src/bin/dhcp4/dhcp4_srv.cc
index 612e8aa58b..f949015c04 100644
--- a/src/bin/dhcp4/dhcp4_srv.cc
+++ b/src/bin/dhcp4/dhcp4_srv.cc
@@ -1117,7 +1117,7 @@ Dhcpv4Srv::runOne() {
}
void
-Dhcpv4Srv::processPacketAndSendResponseNoThrow(Pkt4Ptr& query) {
+Dhcpv4Srv::processPacketAndSendResponseNoThrow(Pkt4Ptr query) {
try {
processPacketAndSendResponse(query);
} catch (const std::exception& e) {
diff --git a/src/bin/dhcp4/dhcp4_srv.h b/src/bin/dhcp4/dhcp4_srv.h
index 0ed55adfe5..0507d73ab5 100644
--- a/src/bin/dhcp4/dhcp4_srv.h
+++ b/src/bin/dhcp4/dhcp4_srv.h
@@ -352,7 +352,7 @@ public:
/// methods, generates appropriate answer, sends the answer to the client.
///
/// @param query A pointer to the packet to be processed.
- void processPacketAndSendResponseNoThrow(Pkt4Ptr& query);
+ void processPacketAndSendResponseNoThrow(Pkt4Ptr query);
/// @brief Process an unparked DHCPv4 packet and sends the response.
///
However, we should probably go over all similar cases, not only this one. I've seen similar issues with other functions.