Obvious fixes: use quota callback which belongs to the socket and...

.. accept connections from the thread (loop) to which listening socket
belongs.

lib/isc/netmgr/tcp.c

@@ -393,7 +393,6 @@ isc_nm_listentcp(isc_nm_t *mgr, isc_nmiface_t *iface,
 		 */
 		sock->pquota = quota;
 	}
-	isc_quota_cb_init(&sock->quotacb, quota_accept_cb, sock);
 
 	ievent = isc__nm_get_netievent_tcplisten(mgr, sock);
 
@@ -601,16 +600,16 @@ isc__nm_async_tcpchildaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 	}
 
 	/* Socket was closed midflight by isc__nm_tcp_shutdown() */
-	if (!isc__nmsocket_active(sock)) {
+	if (inactive(sock)) {
 		result = ISC_R_CANCELED;
-		goto error;
+		goto before_import_error;
 	}
 
 	INSIST(sock->server != NULL);
 
 	if (!isc__nmsocket_active(sock->server)) {
 		result = ISC_R_CANCELED;
-		goto error;
+		goto before_import_error;
 	}
 
 	sock->quota = ievent->quota;
@@ -619,6 +618,8 @@ isc__nm_async_tcpchildaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 	worker = &sock->mgr->workers[isc_nm_tid()];
 	uv_tcp_init(&worker->loop, &sock->uv_handle.tcp);
 
+	/* TODO: we need to close the socket descriptor inside
+	 * ievent->streaminfo properly */
 	r = isc_uv_import(&sock->uv_handle.stream, &ievent->streaminfo);
 	if (r != 0) {
 		isc_log_write(isc_lctx, ISC_LOGCATEGORY_GENERAL,
@@ -672,6 +673,17 @@ isc__nm_async_tcpchildaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 
 	return;
 
+before_import_error:
+	if (ievent->quota != NULL)
+	{
+		isc_quota_detach(&ievent->quota);
+	}
+	failed_accept_cb(sock, result);
+	/* to avoid passing the uninitialised socket to the LibUV */
+	atomic_store(&sock->closed, true);
+	isc__nmsocket_detach(&sock->server);
+	isc__nmsocket_detach(&sock);
+	return;
 error:
 	failed_accept_cb(sock, result);
 	isc__nmsocket_detach(&sock);
@@ -1027,12 +1039,13 @@ quota_accept_cb(isc_quota_t *quota, void *sock0) {
 	isc__netievent_tcpaccept_t *ievent = NULL;
 
 	REQUIRE(VALID_NMSOCK(sock));
+	REQUIRE(sock->server != NULL);
 
 	/*
 	 * Create a tcpaccept event and pass it using the async channel.
 	 */
 	ievent = isc__nm_get_netievent_tcpaccept(sock->mgr, sock, quota);
-	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->tid],
+	isc__nm_enqueue_ievent(&sock->mgr->workers[sock->server->tid],
 			       (isc__netievent_t *)ievent);
 }
 
@@ -1042,11 +1055,13 @@ quota_accept_cb(isc_quota_t *quota, void *sock0) {
 void
 isc__nm_async_tcpaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 	isc__netievent_tcpaccept_t *ievent = (isc__netievent_tcpaccept_t *)ev0;
-	isc_nmsocket_t *sock = ievent->sock;
+	isc_nmsocket_t *sock = ievent->sock, *ssock;
 	isc_result_t result;
 
-	REQUIRE(worker->id == sock->tid);
+	REQUIRE(sock->server != NULL);
+	REQUIRE(worker->id == sock->server->tid);
 
+	ssock = sock->server;
 	result = accept_child_connection(ievent->sock, ievent->quota);
 	if (result != ISC_R_SUCCESS && result != ISC_R_NOCONN) {
 		if ((result != ISC_R_QUOTA && result != ISC_R_SOFTQUOTA) ||
@@ -1059,9 +1074,12 @@ isc__nm_async_tcpaccept(isc__networker_t *worker, isc__netievent_t *ev0) {
 	}
 
 	/*
-	 * The socket was attached just before we called isc_quota_attach_cb().
+	 * The sockets were attached just before we called isc_quota_attach_cb().
 	 */
-	isc__nmsocket_detach(&ievent->sock);
+	/* We cannot use ievent->sock directly because the calling code has
+	 * attached to it and thus expects it to remain not nullified. */
+	isc__nmsocket_detach(&sock);
+	isc__nmsocket_detach(&ssock);
 }
 
 /*
@@ -1081,12 +1099,18 @@ accept_child_connection(isc_nmsocket_t *csock, isc_quota_t *quota) {
 	isc_mem_t *mctx = NULL;
 	int r;
 	isc_result_t result;
+	isc_uv_stream_info_t si;
+	isc_nm_t *mgr = NULL;
 
 	REQUIRE(VALID_NMSOCK(csock));
 
 	if (inactive(csock) || !csock->accepting) {
+		if (quota != NULL) {
+			isc_quota_detach(&quota);
+		}
 		failed_accept_cb(csock, ISC_R_CANCELED);
-		return (ISC_R_CANCELED);
+		result = ISC_R_CANCELED;
+		goto error;
 	}
 
 	isc__nm_incstats(csock->mgr, csock->statsindex[STATID_ACCEPT]);
@@ -1108,38 +1132,50 @@ accept_child_connection(isc_nmsocket_t *csock, isc_quota_t *quota) {
 		if (quota != NULL) {
 			isc_quota_detach(&quota);
 		}
-		return (result);
+		failed_accept_cb(csock, result);
+		goto error;
 	}
 
 	/* We have an accepted TCP socket, pass it to a random worker */
-	event = isc__nm_get_netievent(csock->server->mgr,
-				   netievent_tcpchildaccept);
+
 
 	/* Duplicate the server socket */
-	r = isc_uv_export((uv_stream_t *)uvstream, &event->streaminfo);
+	r = isc_uv_export((uv_stream_t *)uvstream, &si);
 	if (r != 0) {
 		result = isc_errno_toresult(errno);
 		uv_close((uv_handle_t *)uvstream, free_uvtcpt);
+		if (quota != NULL) {
+			isc_quota_detach(&quota);
+		}
 		failed_accept_cb(csock, result);
-		isc__nm_put_netievent(csock->mgr, event);
-		return (result);
+		goto error;
 	}
 
-	event->sock = csock;
-	event->quota = quota;
-
 	uv_close((uv_handle_t *)uvstream, free_uvtcpt);
 
+	isc_nm_attach(csock->mgr, &mgr);
 	if (csock->tid == isc_nm_tid()) {
-		isc__nm_async_tcpchildaccept(&csock->mgr->workers[csock->tid],
+		event = isc__nm_get_netievent(mgr, netievent_tcpchildaccept);
+		event->sock = csock;
+		event->quota = quota;
+		isc__nm_async_tcpchildaccept(&mgr->workers[csock->tid],
 					     (isc__netievent_t *)event);
-		isc__nm_put_netievent(csock->mgr, event);
+		isc__nm_put_netievent(mgr, event);
 	} else {
-		isc__nm_enqueue_ievent(&csock->mgr->workers[csock->tid],
+		event = isc__nm_get_netievent_tcpchildaccept(mgr, csock, si,
+							     quota);
+		isc__nm_enqueue_ievent(&mgr->workers[csock->tid],
 				       (isc__netievent_t *)event);
 	}
+	isc_nm_detach(&mgr);
 
 	return (ISC_R_SUCCESS);
+error:
+	/* to avoid passing the uninitialised socket to the LibUV */
+	atomic_store(&csock->closed, true);
+	isc__nmsocket_detach(&csock->server);
+	isc__nmsocket_detach(&csock);
+	return result;
 }
 
 static isc_result_t
@@ -1168,14 +1204,20 @@ accept_connection(isc_nmsocket_t *ssock) {
 	csock->accepting = true;
 
 	if (ssock->pquota != NULL) {
+		isc_nmsocket_t *tmp_csock = NULL, *tmp_ssock = NULL;
 		isc_quota_cb_init(&csock->quotacb, quota_accept_cb, csock);
+		/* keep it until the quotacb concludes */
+		isc__nmsocket_attach(csock, &tmp_csock);
+		isc__nmsocket_attach(ssock, &tmp_ssock);
 		result = isc_quota_attach_cb(ssock->pquota, &quota,
-					     &ssock->quotacb);
+					     &csock->quotacb);
 		if (result == ISC_R_QUOTA) {
 			isc__nm_incstats(ssock->mgr,
 					 ssock->statsindex[STATID_ACCEPTFAIL]);
 			return (result);
 		}
+		isc__nmsocket_detach(&tmp_ssock);
+		isc__nmsocket_detach(&tmp_csock);
 	}
 
 	return (accept_child_connection(csock, quota));