socket.c 104 KB
Newer Older
Danny Mayer's avatar
Danny Mayer committed
1
/*
Tinderbox User's avatar
Tinderbox User committed
2
 * Copyright (C) 2004-2014  Internet Systems Consortium, Inc. ("ISC")
Mark Andrews's avatar
Mark Andrews committed
3
 * Copyright (C) 2000-2003  Internet Software Consortium.
Danny Mayer's avatar
Danny Mayer committed
4
 *
Automatic Updater's avatar
Automatic Updater committed
5
 * Permission to use, copy, modify, and/or distribute this software for any
Danny Mayer's avatar
Danny Mayer committed
6 7 8
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
Mark Andrews's avatar
Mark Andrews committed
9 10 11 12 13 14 15
 * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
 * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
 * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
Danny Mayer's avatar
Danny Mayer committed
16 17
 */

Mark Andrews's avatar
Mark Andrews committed
18
/* $Id$ */
19

20 21
/* This code uses functions which are only available on Server 2003 and
 * higher, and Windows XP and higher.
22 23 24
 *
 * This code is by nature multithreaded and takes advantage of various
 * features to pass on information through the completion port for
25 26
 * when I/O is completed.  All sends, receives, accepts, and connects are
 * completed through the completion port.
27
 *
28 29 30
 * The number of Completion Port Worker threads used is the total number
 * of CPU's + 1. This increases the likelihood that a Worker Thread is
 * available for processing a completed request.
31 32 33
 *
 * XXXPDM 5 August, 2002
 */
Danny Mayer's avatar
Danny Mayer committed
34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50

#define MAKE_EXTERNAL 1
#include <config.h>

#include <sys/types.h>

#ifndef _WINSOCKAPI_
#define _WINSOCKAPI_   /* Prevent inclusion of winsock.h in windows.h */
#endif

#include <errno.h>
#include <stddef.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <io.h>
#include <fcntl.h>
51
#include <process.h>
Danny Mayer's avatar
Danny Mayer committed
52

53
#include <isc/app.h>
Danny Mayer's avatar
Danny Mayer committed
54 55 56 57 58 59 60 61 62
#include <isc/buffer.h>
#include <isc/bufferlist.h>
#include <isc/condition.h>
#include <isc/list.h>
#include <isc/log.h>
#include <isc/mem.h>
#include <isc/msgs.h>
#include <isc/mutex.h>
#include <isc/net.h>
63
#include <isc/once.h>
64
#include <isc/os.h>
Danny Mayer's avatar
Danny Mayer committed
65 66 67 68
#include <isc/platform.h>
#include <isc/print.h>
#include <isc/region.h>
#include <isc/socket.h>
69
#include <isc/stats.h>
70 71
#include <isc/strerror.h>
#include <isc/syslog.h>
Danny Mayer's avatar
Danny Mayer committed
72 73 74
#include <isc/task.h>
#include <isc/thread.h>
#include <isc/util.h>
75
#include <isc/win32os.h>
Danny Mayer's avatar
Danny Mayer committed
76

77 78
#include <mswsock.h>

79
#include "errno2result.h"
80

Evan Hunt's avatar
Evan Hunt committed
81 82 83 84 85 86 87
/*
 * Set by the -T dscp option on the command line. If set to a value
 * other than -1, we check to make sure DSCP values match it, and
 * assert if not.
 */
int isc_dscp_check_value = -1;

88 89 90 91 92 93 94 95 96 97 98
/*
 * How in the world can Microsoft exist with APIs like this?
 * We can't actually call this directly, because it turns out
 * no library exports this function.  Instead, we need to
 * issue a runtime call to get the address.
 */
LPFN_CONNECTEX ISCConnectEx;
LPFN_ACCEPTEX ISCAcceptEx;
LPFN_GETACCEPTEXSOCKADDRS ISCGetAcceptExSockaddrs;

/*
Francis Dupont's avatar
Francis Dupont committed
99
 * Run expensive internal consistency checks.
100 101 102 103 104 105 106 107
 */
#ifdef ISC_SOCKET_CONSISTENCY_CHECKS
#define CONSISTENT(sock) consistent(sock)
#else
#define CONSISTENT(sock) do {} while (0)
#endif
static void consistent(isc_socket_t *sock);

108 109 110 111 112 113 114
/*
 * Define this macro to control the behavior of connection
 * resets on UDP sockets.  See Microsoft KnowledgeBase Article Q263823
 * for details.
 * NOTE: This requires that Windows 2000 systems install Service Pack 2
 * or later.
 */
115 116
#ifndef SIO_UDP_CONNRESET
#define SIO_UDP_CONNRESET _WSAIOW(IOC_VENDOR,12)
117 118
#endif

Danny Mayer's avatar
Danny Mayer committed
119 120 121 122 123 124 125 126 127 128 129 130
/*
 * Some systems define the socket length argument as an int, some as size_t,
 * some as socklen_t.  This is here so it can be easily changed if needed.
 */
#ifndef ISC_SOCKADDR_LEN_T
#define ISC_SOCKADDR_LEN_T unsigned int
#endif

/*
 * Define what the possible "soft" errors can be.  These are non-fatal returns
 * of various network related functions, like recv() and so on.
 */
131
#define SOFT_ERROR(e)	((e) == WSAEINTR || \
132
			 (e) == WSAEWOULDBLOCK || \
133 134 135
			 (e) == EWOULDBLOCK || \
			 (e) == EINTR || \
			 (e) == EAGAIN || \
Danny Mayer's avatar
Danny Mayer committed
136 137
			 (e) == 0)

138 139 140 141 142 143 144
/*
 * Pending errors are not really errors and should be
 * kept separate
 */
#define PENDING_ERROR(e) ((e) == WSA_IO_PENDING || (e) == 0)

#define DOIO_SUCCESS	  0       /* i/o ok, event sent */
145 146 147
#define DOIO_SOFT	  1       /* i/o ok, soft error, no event sent */
#define DOIO_HARD	  2       /* i/o error, event sent */
#define DOIO_EOF	  3       /* EOF, no event sent */
148
#define DOIO_PENDING	  4       /* status when i/o is in process */
149
#define DOIO_NEEDMORE	  5       /* IO was processed, but we need more due to minimum */
150

Danny Mayer's avatar
Danny Mayer committed
151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173
#define DLVL(x) ISC_LOGCATEGORY_GENERAL, ISC_LOGMODULE_SOCKET, ISC_LOG_DEBUG(x)

/*
 * DLVL(90)  --  Function entry/exit and other tracing.
 * DLVL(70)  --  Socket "correctness" -- including returning of events, etc.
 * DLVL(60)  --  Socket data send/receive
 * DLVL(50)  --  Event tracing, including receiving/sending completion events.
 * DLVL(20)  --  Socket creation/destruction.
 */
#define TRACE_LEVEL		90
#define CORRECTNESS_LEVEL	70
#define IOEVENT_LEVEL		60
#define EVENT_LEVEL		50
#define CREATION_LEVEL		20

#define TRACE		DLVL(TRACE_LEVEL)
#define CORRECTNESS	DLVL(CORRECTNESS_LEVEL)
#define IOEVENT		DLVL(IOEVENT_LEVEL)
#define EVENT		DLVL(EVENT_LEVEL)
#define CREATION	DLVL(CREATION_LEVEL)

typedef isc_event_t intev_t;

174 175 176 177 178 179 180 181 182 183 184 185 186
/*
 * Socket State
 */
enum {
  SOCK_INITIALIZED,	/* Socket Initialized */
  SOCK_OPEN,		/* Socket opened but nothing yet to do */
  SOCK_DATA,		/* Socket sending or receiving data */
  SOCK_LISTEN,		/* TCP Socket listening for connects */
  SOCK_ACCEPT,		/* TCP socket is waiting to accept */
  SOCK_CONNECT,		/* TCP Socket connecting */
  SOCK_CLOSED,		/* Socket has been closed */
};

187 188
#define SOCKET_MAGIC		ISC_MAGIC('I', 'O', 'i', 'o')
#define VALID_SOCKET(t)		ISC_MAGIC_VALID(t, SOCKET_MAGIC)
Danny Mayer's avatar
Danny Mayer committed
189 190 191 192 193 194 195 196 197 198 199 200 201 202

/*
 * IPv6 control information.  If the socket is an IPv6 socket we want
 * to collect the destination address and interface so the client can
 * set them on outgoing packets.
 */
#ifdef ISC_PLATFORM_HAVEIPV6
#ifndef USE_CMSG
#define USE_CMSG	1
#endif
#endif

/*
 * We really  don't want to try and use these control messages. Win32
203
 * doesn't have this mechanism before XP.
Danny Mayer's avatar
Danny Mayer committed
204 205 206 207 208 209 210 211
 */
#undef USE_CMSG

/*
 * Message header for recvmsg and sendmsg calls.
 * Used value-result for recvmsg, value only for sendmsg.
 */
struct msghdr {
212 213
	SOCKADDR_STORAGE to_addr;	/* UDP send/recv address */
	int      to_addr_len;		/* length of the address */
Automatic Updater's avatar
Automatic Updater committed
214 215 216 217
	WSABUF  *msg_iov;		/* scatter/gather array */
	u_int   msg_iovlen;             /* # elements in msg_iov */
	void	*msg_control;           /* ancillary data, see below */
	u_int   msg_controllen;         /* ancillary data buffer len */
218
	u_int	msg_totallen;		/* total length of this message */
Danny Mayer's avatar
Danny Mayer committed
219
} msghdr;
Automatic Updater's avatar
Automatic Updater committed
220

221 222
/*
 * The size to raise the receive buffer to.
223 224 225
 */
#define RCVBUFSIZE (32*1024)

226
/*
227 228
 * The number of times a send operation is repeated if the result
 * is WSAEINTR.
229 230 231
 */
#define NRETRIES 10

Danny Mayer's avatar
Danny Mayer committed
232 233 234 235 236 237
struct isc_socket {
	/* Not locked. */
	unsigned int		magic;
	isc_socketmgr_t	       *manager;
	isc_mutex_t		lock;
	isc_sockettype_t	type;
238

239 240
	/* Pointers to scatter/gather buffers */
	WSABUF			iov[ISC_SOCKET_MAXSCATTERGATHER];
241

Danny Mayer's avatar
Danny Mayer committed
242 243
	/* Locked by socket lock. */
	ISC_LINK(isc_socket_t)	link;
244 245 246
	unsigned int		references; /* EXTERNAL references */
	SOCKET			fd;	/* file handle */
	int			pf;	/* protocol family */
247 248 249
	char			name[16];
	void *			tag;

250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265
	/*
	 * Each recv() call uses this buffer.  It is a per-socket receive
	 * buffer that allows us to decouple the system recv() from the
	 * recv_list done events.  This means the items on the recv_list
	 * can be removed without having to cancel pending system recv()
	 * calls.  It also allows us to read-ahead in some cases.
	 */
	struct {
		SOCKADDR_STORAGE	from_addr;	   // UDP send/recv address
		int		from_addr_len;	   // length of the address
		char		*base;		   // the base of the buffer
		char		*consume_position; // where to start copying data from next
		unsigned int	len;		   // the actual size of this buffer
		unsigned int	remaining;	   // the number of bytes remaining
	} recvbuf;

Danny Mayer's avatar
Danny Mayer committed
266 267 268
	ISC_LIST(isc_socketevent_t)		send_list;
	ISC_LIST(isc_socketevent_t)		recv_list;
	ISC_LIST(isc_socket_newconnev_t)	accept_list;
269
	ISC_LIST(isc_socket_connev_t)		connect_list;
Danny Mayer's avatar
Danny Mayer committed
270 271 272

	isc_sockaddr_t		address;  /* remote address */

273
	unsigned int		listener : 1,	/* listener socket */
Danny Mayer's avatar
Danny Mayer committed
274
				connected : 1,
275
				pending_connect : 1, /* connect pending */
276 277
				bound : 1,	/* bound to local addr */
				dupped : 1;     /* created by isc_socket_dup() */
278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296
	unsigned int		pending_iocp;	/* Should equal the counters below. Debug. */
	unsigned int		pending_recv;  /* Number of outstanding recv() calls. */
	unsigned int		pending_send;  /* Number of outstanding send() calls. */
	unsigned int		pending_accept; /* Number of outstanding accept() calls. */
	unsigned int		state; /* Socket state. Debugging and consistency checking. */
	int			state_lineno;  /* line which last touched state */
};

#define _set_state(sock, _state) do { (sock)->state = (_state); (sock)->state_lineno = __LINE__; } while (0)

/*
 * Buffer structure
 */
typedef struct buflist buflist_t;

struct buflist {
	void			*buf;
	unsigned int		buflen;
	ISC_LINK(buflist_t)	link;
Danny Mayer's avatar
Danny Mayer committed
297 298
};

299 300 301 302 303 304
/*
 * I/O Completion ports Info structures
 */

static HANDLE hHeapHandle = NULL;
typedef struct IoCompletionInfo {
305 306 307 308 309 310 311 312 313
	OVERLAPPED		overlapped;
	isc_socketevent_t	*dev;  /* send()/recv() done event */
	isc_socket_connev_t	*cdev; /* connect() done event */
	isc_socket_newconnev_t	*adev; /* accept() done event */
	void			*acceptbuffer;
	DWORD			received_bytes;
	int			request_type;
	struct msghdr		messagehdr;
	ISC_LIST(buflist_t)	bufferlist;	/*%< list of buffers */
314 315 316 317
} IoCompletionInfo;

/*
 * Define a maximum number of I/O Completion Port worker threads
318 319
 * to handle the load on the Completion Port. The actual number
 * used is the number of CPU's + 1.
320 321 322
 */
#define MAX_IOCPTHREADS 20

323 324 325
#define SOCKET_MANAGER_MAGIC	ISC_MAGIC('I', 'O', 'm', 'g')
#define VALID_MANAGER(m)	ISC_MAGIC_VALID(m, SOCKET_MANAGER_MAGIC)

Danny Mayer's avatar
Danny Mayer committed
326 327
struct isc_socketmgr {
	/* Not locked. */
328 329 330
	unsigned int			magic;
	isc_mem_t		       *mctx;
	isc_mutex_t			lock;
331 332
	isc_stats_t		       *stats;

Danny Mayer's avatar
Danny Mayer committed
333
	/* Locked by manager lock. */
334 335 336 337 338 339 340
	ISC_LIST(isc_socket_t)		socklist;
	isc_boolean_t			bShutdown;
	isc_condition_t			shutdown_ok;
	HANDLE				hIoCompletionPort;
	int				maxIOCPThreads;
	HANDLE				hIOCPThreads[MAX_IOCPTHREADS];
	DWORD				dwIOCPThreadIds[MAX_IOCPTHREADS];
341 342 343 344 345 346 347 348 349 350 351 352 353 354

	/*
	 * Debugging.
	 * Modified by InterlockedIncrement() and InterlockedDecrement()
	 */
	LONG				totalSockets;
	LONG				iocp_total;
};

enum {
	SOCKET_RECV,
	SOCKET_SEND,
	SOCKET_ACCEPT,
	SOCKET_CONNECT
Danny Mayer's avatar
Danny Mayer committed
355 356 357 358 359 360
};

/*
 * send() and recv() iovec counts
 */
#define MAXSCATTERGATHER_SEND	(ISC_SOCKET_MAXSCATTERGATHER)
361
#define MAXSCATTERGATHER_RECV	(ISC_SOCKET_MAXSCATTERGATHER)
Danny Mayer's avatar
Danny Mayer committed
362

363 364 365 366
static isc_result_t socket_create(isc_socketmgr_t *manager0, int pf,
				  isc_sockettype_t type,
				  isc_socket_t **socketp,
				  isc_socket_t *dup_socket);
367
static isc_threadresult_t WINAPI SocketIoThread(LPVOID ThreadContext);
368 369 370 371 372 373 374 375 376 377
static void maybe_free_socket(isc_socket_t **, int);
static void free_socket(isc_socket_t **, int);
static isc_boolean_t senddone_is_active(isc_socket_t *sock, isc_socketevent_t *dev);
static isc_boolean_t acceptdone_is_active(isc_socket_t *sock, isc_socket_newconnev_t *dev);
static isc_boolean_t connectdone_is_active(isc_socket_t *sock, isc_socket_connev_t *dev);
static void send_recvdone_event(isc_socket_t *sock, isc_socketevent_t **dev);
static void send_senddone_event(isc_socket_t *sock, isc_socketevent_t **dev);
static void send_acceptdone_event(isc_socket_t *sock, isc_socket_newconnev_t **adev);
static void send_connectdone_event(isc_socket_t *sock, isc_socket_connev_t **cdev);
static void send_recvdone_abort(isc_socket_t *sock, isc_result_t result);
378
static void send_connectdone_abort(isc_socket_t *sock, isc_result_t result);
379 380
static void queue_receive_event(isc_socket_t *sock, isc_task_t *task, isc_socketevent_t *dev);
static void queue_receive_request(isc_socket_t *sock);
381 382

/*
383
 * This is used to dump the contents of the sock structure
384 385 386 387 388 389 390 391
 * You should make sure that the sock is locked before
 * dumping it. Since the code uses simple printf() statements
 * it should only be used interactively.
 */
void
sock_dump(isc_socket_t *sock) {
	isc_socketevent_t *ldev;
	isc_socket_newconnev_t *ndev;
392
	isc_socket_connev_t *cdev;
393 394

#if 0
395
	isc_sockaddr_t addr;
396 397 398 399 400 401 402 403 404 405 406 407 408
	char socktext[ISC_SOCKADDR_FORMATSIZE];
	isc_result_t result;

	result = isc_socket_getpeername(sock, &addr);
	if (result == ISC_R_SUCCESS) {
		isc_sockaddr_format(&addr, socktext, sizeof(socktext));
		printf("Remote Socket: %s\n", socktext);
	}
	result = isc_socket_getsockname(sock, &addr);
	if (result == ISC_R_SUCCESS) {
		isc_sockaddr_format(&addr, socktext, sizeof(socktext));
		printf("This Socket: %s\n", socktext);
	}
409
#endif
410 411 412 413 414

	printf("\n\t\tSock Dump\n");
	printf("\t\tfd: %u\n", sock->fd);
	printf("\t\treferences: %d\n", sock->references);
	printf("\t\tpending_accept: %d\n", sock->pending_accept);
415
	printf("\t\tconnecting: %d\n", sock->pending_connect);
416 417
	printf("\t\tconnected: %d\n", sock->connected);
	printf("\t\tbound: %d\n", sock->bound);
418
	printf("\t\tpending_iocp: %d\n", sock->pending_iocp);
419 420 421 422 423 424 425 426
	printf("\t\tsocket type: %d\n", sock->type);

	printf("\n\t\tSock Recv List\n");
	ldev = ISC_LIST_HEAD(sock->recv_list);
	while (ldev != NULL) {
		printf("\t\tdev: %p\n", ldev);
		ldev = ISC_LIST_NEXT(ldev, ev_link);
	}
427

428 429 430 431 432 433
	printf("\n\t\tSock Send List\n");
	ldev = ISC_LIST_HEAD(sock->send_list);
	while (ldev != NULL) {
		printf("\t\tdev: %p\n", ldev);
		ldev = ISC_LIST_NEXT(ldev, ev_link);
	}
434

435 436 437 438 439 440
	printf("\n\t\tSock Accept List\n");
	ndev = ISC_LIST_HEAD(sock->accept_list);
	while (ndev != NULL) {
		printf("\t\tdev: %p\n", ldev);
		ndev = ISC_LIST_NEXT(ndev, ev_link);
	}
441 442 443 444 445 446 447

	printf("\n\t\tSock Connect List\n");
	cdev = ISC_LIST_HEAD(sock->connect_list);
	while (cdev != NULL) {
		printf("\t\tdev: %p\n", cdev);
		cdev = ISC_LIST_NEXT(cdev, ev_link);
	}
448
}
449 450 451 452 453 454

static void
socket_log(int lineno, isc_socket_t *sock, isc_sockaddr_t *address,
	   isc_logcategory_t *category, isc_logmodule_t *module, int level,
	   isc_msgcat_t *msgcat, int msgset, int message,
	   const char *fmt, ...) ISC_FORMAT_PRINTF(9, 10);
455 456 457 458 459 460 461 462 463 464

/*  This function will add an entry to the I/O completion port
 *  that will signal the I/O thread to exit (gracefully)
 */
static void
signal_iocompletionport_exit(isc_socketmgr_t *manager) {
	int i;
	int errval;
	char strbuf[ISC_STRERRORSIZE];

465
	REQUIRE(VALID_MANAGER(manager));
466
	for (i = 0; i < manager->maxIOCPThreads; i++) {
467 468
		if (!PostQueuedCompletionStatus(manager->hIoCompletionPort,
						0, 0, 0)) {
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486
			errval = GetLastError();
			isc__strerror(errval, strbuf, sizeof(strbuf));
			FATAL_ERROR(__FILE__, __LINE__,
				isc_msgcat_get(isc_msgcat, ISC_MSGSET_SOCKET,
				ISC_MSG_FAILED,
				"Can't request service thread to exit: %s"),
				strbuf);
		}
	}
}

/*
 * Create the worker threads for the I/O Completion Port
 */
void
iocompletionport_createthreads(int total_threads, isc_socketmgr_t *manager) {
	int errval;
	char strbuf[ISC_STRERRORSIZE];
487
	int i;
488

489 490
	INSIST(total_threads > 0);
	REQUIRE(VALID_MANAGER(manager));
491 492 493 494
	/*
	 * We need at least one
	 */
	for (i = 0; i < total_threads; i++) {
495
		manager->hIOCPThreads[i] = CreateThread(NULL, 0, SocketIoThread,
496 497
						manager, 0,
						&manager->dwIOCPThreadIds[i]);
498
		if (manager->hIOCPThreads[i] == NULL) {
499 500 501 502 503 504 505
			errval = GetLastError();
			isc__strerror(errval, strbuf, sizeof(strbuf));
			FATAL_ERROR(__FILE__, __LINE__,
				isc_msgcat_get(isc_msgcat, ISC_MSGSET_SOCKET,
				ISC_MSG_FAILED,
				"Can't create IOCP thread: %s"),
				strbuf);
506
			exit(1);
507 508 509 510
		}
	}
}

511
/*
512
 *  Create/initialise the I/O completion port
513
 */
514 515 516 517
void
iocompletionport_init(isc_socketmgr_t *manager) {
	int errval;
	char strbuf[ISC_STRERRORSIZE];
518 519

	REQUIRE(VALID_MANAGER(manager));
520 521
	/*
	 * Create a private heap to handle the socket overlapped structure
Francis Dupont's avatar
Francis Dupont committed
522
	 * The minimum number of structures is 10, there is no maximum
523
	 */
524 525 526 527 528 529 530 531 532 533 534 535 536 537
	hHeapHandle = HeapCreate(0, 10 * sizeof(IoCompletionInfo), 0);
	if (hHeapHandle == NULL) {
		errval = GetLastError();
		isc__strerror(errval, strbuf, sizeof(strbuf));
		FATAL_ERROR(__FILE__, __LINE__,
			    isc_msgcat_get(isc_msgcat, ISC_MSGSET_SOCKET,
					   ISC_MSG_FAILED,
					   "HeapCreate() failed during "
					   "initialization: %s"),
			    strbuf);
		exit(1);
	}

	manager->maxIOCPThreads = min(isc_os_ncpus() + 1, MAX_IOCPTHREADS);
538 539 540

	/* Now Create the Completion Port */
	manager->hIoCompletionPort = CreateIoCompletionPort(
541 542
			INVALID_HANDLE_VALUE, NULL,
			0, manager->maxIOCPThreads);
543 544 545 546 547 548 549 550 551 552 553
	if (manager->hIoCompletionPort == NULL) {
		errval = GetLastError();
		isc__strerror(errval, strbuf, sizeof(strbuf));
		FATAL_ERROR(__FILE__, __LINE__,
				isc_msgcat_get(isc_msgcat, ISC_MSGSET_SOCKET,
				ISC_MSG_FAILED,
				"CreateIoCompletionPort() failed "
				"during initialization: %s"),
				strbuf);
		exit(1);
	}
554

555 556
	/*
	 * Worker threads for servicing the I/O
Automatic Updater's avatar
Automatic Updater committed
557
	 */
558 559
	iocompletionport_createthreads(manager->maxIOCPThreads, manager);
}
560

561
/*
562 563
 * Associate a socket with an IO Completion Port.  This allows us to queue events for it
 * and have our worker pool of threads process them.
564
 */
565
void
566 567
iocompletionport_update(isc_socket_t *sock) {
	HANDLE hiocp;
568
	char strbuf[ISC_STRERRORSIZE];
569

570
	REQUIRE(VALID_SOCKET(sock));
571

572 573 574 575 576 577
	hiocp = CreateIoCompletionPort((HANDLE)sock->fd,
		sock->manager->hIoCompletionPort, (ULONG_PTR)sock, 0);

	if (hiocp == NULL) {
		DWORD errval = GetLastError();
		isc__strerror(errval, strbuf, sizeof(strbuf));
578 579 580 581
		isc_log_iwrite(isc_lctx,
				ISC_LOGCATEGORY_GENERAL,
				ISC_LOGMODULE_SOCKET, ISC_LOG_ERROR,
				isc_msgcat, ISC_MSGSET_SOCKET,
Automatic Updater's avatar
Automatic Updater committed
582
				ISC_MSG_TOOMANYHANDLES,
583 584 585 586 587 588 589 590 591 592 593 594 595 596 597
				"iocompletionport_update: failed to open"
				" io completion port: %s",
				strbuf);

		/* XXXMLG temporary hack to make failures detected.
		 * This function should return errors to the caller, not
		 * exit here.
		 */
		FATAL_ERROR(__FILE__, __LINE__,
				isc_msgcat_get(isc_msgcat, ISC_MSGSET_SOCKET,
				ISC_MSG_FAILED,
				"CreateIoCompletionPort() failed "
				"during initialization: %s"),
				strbuf);
		exit(1);
598 599
	}

600
	InterlockedIncrement(&sock->manager->iocp_total);
601
}
602

603
/*
604 605 606
 * Routine to cleanup and then close the socket.
 * Only close the socket here if it is NOT associated
 * with an event, otherwise the WSAWaitForMultipleEvents
607
 * may fail due to the fact that the Wait should not
608 609
 * be running while closing an event or a socket.
 * The socket is locked before calling this function
610
 */
611 612
void
socket_close(isc_socket_t *sock) {
613

614
	REQUIRE(sock != NULL);
615

616 617 618 619 620
	if (sock->fd != INVALID_SOCKET) {
		closesocket(sock->fd);
		sock->fd = INVALID_SOCKET;
		_set_state(sock, SOCK_CLOSED);
		InterlockedDecrement(&sock->manager->totalSockets);
621
	}
622
}
623

624 625
static isc_once_t initialise_once = ISC_ONCE_INIT;
static isc_boolean_t initialised = ISC_FALSE;
626

627 628 629 630 631 632 633 634 635 636
static void
initialise(void) {
	WORD wVersionRequested;
	WSADATA wsaData;
	int err;
	SOCKET sock;
	GUID GUIDConnectEx = WSAID_CONNECTEX;
	GUID GUIDAcceptEx = WSAID_ACCEPTEX;
	GUID GUIDGetAcceptExSockaddrs = WSAID_GETACCEPTEXSOCKADDRS;
	DWORD dwBytes;
637

638 639 640 641 642 643 644 645 646 647 648 649
	/* Need Winsock 2.2 or better */
	wVersionRequested = MAKEWORD(2, 2);

	err = WSAStartup(wVersionRequested, &wsaData);
	if (err != 0) {
		char strbuf[ISC_STRERRORSIZE];
		isc__strerror(err, strbuf, sizeof(strbuf));
		FATAL_ERROR(__FILE__, __LINE__, "WSAStartup() %s: %s",
			    isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
					   ISC_MSG_FAILED, "failed"),
			    strbuf);
		exit(1);
650 651
	}
	/*
652 653 654 655
	 * The following APIs do not exist as functions in a library, but we must
	 * ask winsock for them.  They are "extensions" -- but why they cannot be
	 * actual functions is beyond me.  So, ask winsock for the pointers to the
	 * functions we need.
656
	 */
657 658 659 660 661 662 663
	sock = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
	INSIST(sock != INVALID_SOCKET);
	err = WSAIoctl(sock,  SIO_GET_EXTENSION_FUNCTION_POINTER,
		 &GUIDConnectEx, sizeof(GUIDConnectEx),
		 &ISCConnectEx, sizeof(ISCConnectEx),
		 &dwBytes, NULL, NULL);
	INSIST(err == 0);
664

665 666 667 668 669
	err = WSAIoctl(sock,  SIO_GET_EXTENSION_FUNCTION_POINTER,
		 &GUIDAcceptEx, sizeof(GUIDAcceptEx),
		 &ISCAcceptEx, sizeof(ISCAcceptEx),
		 &dwBytes, NULL, NULL);
	INSIST(err == 0);
670

671 672 673 674 675
	err = WSAIoctl(sock,  SIO_GET_EXTENSION_FUNCTION_POINTER,
		 &GUIDGetAcceptExSockaddrs, sizeof(GUIDGetAcceptExSockaddrs),
		 &ISCGetAcceptExSockaddrs, sizeof(ISCGetAcceptExSockaddrs),
		 &dwBytes, NULL, NULL);
	INSIST(err == 0);
676

677
	closesocket(sock);
678

679
	initialised = ISC_TRUE;
680
}
681

682
/*
683
 * Initialize socket services
684
 */
685 686 687 688 689 690 691 692 693 694 695
void
InitSockets(void) {
	RUNTIME_CHECK(isc_once_do(&initialise_once,
				  initialise) == ISC_R_SUCCESS);
	if (!initialised)
		exit(1);
}

int
internal_sendmsg(isc_socket_t *sock, IoCompletionInfo *lpo,
		 struct msghdr *messagehdr, int flags, int *Error)
696
{
697 698 699 700
	int Result;
	DWORD BytesSent;
	DWORD Flags = flags;
	int total_sent;
701

702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725
	*Error = 0;
	Result = WSASendTo(sock->fd, messagehdr->msg_iov,
			   messagehdr->msg_iovlen, &BytesSent,
			   Flags, (SOCKADDR *)&messagehdr->to_addr,
			   messagehdr->to_addr_len, (LPWSAOVERLAPPED)lpo,
			   NULL);

	total_sent = (int)BytesSent;

	/* Check for errors.*/
	if (Result == SOCKET_ERROR) {
		*Error = WSAGetLastError();

		switch (*Error) {
		case WSA_IO_INCOMPLETE:
		case WSA_WAIT_IO_COMPLETION:
		case WSA_IO_PENDING:
		case NO_ERROR:		/* Strange, but okay */
			sock->pending_iocp++;
			sock->pending_send++;
			break;

		default:
			return (-1);
726 727
			break;
		}
728 729 730
	} else {
		sock->pending_iocp++;
		sock->pending_send++;
731
	}
732

733 734 735 736 737 738 739 740 741 742 743 744
	if (lpo != NULL)
		return (0);
	else
		return (total_sent);
}

static void
queue_receive_request(isc_socket_t *sock) {
	DWORD Flags = 0;
	DWORD NumBytes = 0;
	int Result;
	int Error;
745
	int need_retry;
746
	WSABUF iov[1];
747
	IoCompletionInfo *lpo = NULL;
748 749
	isc_result_t isc_result;

750 751 752
 retry:
	need_retry = ISC_FALSE;

753
	/*
754
	 * If we already have a receive pending, do nothing.
755
	 */
756 757 758
	if (sock->pending_recv > 0) {
		if (lpo != NULL)
			HeapFree(hHeapHandle, 0, lpo);
759
		return;
760
	}
761

762 763 764
	/*
	 * If no one is waiting, do nothing.
	 */
765 766 767
	if (ISC_LIST_EMPTY(sock->recv_list)) {
		if (lpo != NULL)
			HeapFree(hHeapHandle, 0, lpo);
768
		return;
769
	}
770

771 772
	INSIST(sock->recvbuf.remaining == 0);
	INSIST(sock->fd != INVALID_SOCKET);
773

774 775
	iov[0].len = sock->recvbuf.len;
	iov[0].buf = sock->recvbuf.base;
776

777 778 779 780 781 782 783
	if (lpo == NULL) {
		lpo = (IoCompletionInfo *)HeapAlloc(hHeapHandle,
						    HEAP_ZERO_MEMORY,
						    sizeof(IoCompletionInfo));
		RUNTIME_CHECK(lpo != NULL);
	} else
		ZeroMemory(lpo, sizeof(IoCompletionInfo));
784
	lpo->request_type = SOCKET_RECV;
785

786
	sock->recvbuf.from_addr_len = sizeof(sock->recvbuf.from_addr);
787

788 789 790 791 792 793
	Error = 0;
	Result = WSARecvFrom((SOCKET)sock->fd, iov, 1,
			     &NumBytes, &Flags,
			     (SOCKADDR *)&sock->recvbuf.from_addr,
			     &sock->recvbuf.from_addr_len,
			     (LPWSAOVERLAPPED)lpo, NULL);
794

795 796 797
	/* Check for errors. */
	if (Result == SOCKET_ERROR) {
		Error = WSAGetLastError();
798

799 800 801 802 803
		switch (Error) {
		case WSA_IO_PENDING:
			sock->pending_iocp++;
			sock->pending_recv++;
			break;
804

805
		/* direct error: no completion event */
806 807 808
		case ERROR_HOST_UNREACHABLE:
		case WSAENETRESET:
		case WSAECONNRESET:
809 810 811
			if (!sock->connected) {
				/* soft error */
				need_retry = ISC_TRUE;
812
				break;
813 814
			}
			/* FALLTHROUGH */
815

816 817
		default:
			isc_result = isc__errno2result(Error);
818
			if (isc_result == ISC_R_UNEXPECTED)
819 820 821 822
				UNEXPECTED_ERROR(__FILE__, __LINE__,
					"WSARecvFrom: Windows error code: %d, isc result %d",
					Error, isc_result);
			send_recvdone_abort(sock, isc_result);
823 824
			HeapFree(hHeapHandle, 0, lpo);
			lpo = NULL;
825 826
			break;
		}
827 828 829 830 831 832 833
	} else {
		/*
		 * The recv() finished immediately, but we will still get
		 * a completion event.  Rather than duplicate code, let
		 * that thread handle sending the data along its way.
		 */
		sock->pending_iocp++;
834
		sock->pending_recv++;
835
	}
836

837 838 839 840 841 842 843
	socket_log(__LINE__, sock, NULL, IOEVENT,
		   isc_msgcat, ISC_MSGSET_SOCKET,
		   ISC_MSG_DOIORECV,
		   "queue_io_request: fd %d result %d error %d",
		   sock->fd, Result, Error);

	CONSISTENT(sock);
844

845 846
	if (need_retry)
		goto retry;
847
}
Danny Mayer's avatar
Danny Mayer committed
848 849

static void
850 851 852
manager_log(isc_socketmgr_t *sockmgr, isc_logcategory_t *category,
	    isc_logmodule_t *module, int level, const char *fmt, ...)
{
Danny Mayer's avatar
Danny Mayer committed
853 854 855
	char msgbuf[2048];
	va_list ap;

856
	if (!isc_log_wouldlog(isc_lctx, level))
Danny Mayer's avatar
Danny Mayer committed
857 858 859 860 861 862 863 864 865 866
		return;

	va_start(ap, fmt);
	vsnprintf(msgbuf, sizeof(msgbuf), fmt, ap);
	va_end(ap);

	isc_log_write(isc_lctx, category, module, level,
		      "sockmgr %p: %s", sockmgr, msgbuf);
}

867
static void
868
socket_log(int lineno, isc_socket_t *sock, isc_sockaddr_t *address,
Danny Mayer's avatar
Danny Mayer committed
869 870
	   isc_logcategory_t *category, isc_logmodule_t *module, int level,
	   isc_msgcat_t *msgcat, int msgset, int message,
871 872
	   const char *fmt, ...)
{
Danny Mayer's avatar
Danny Mayer committed
873 874 875 876
	char msgbuf[2048];
	char peerbuf[256];
	va_list ap;

877 878

	if (!isc_log_wouldlog(isc_lctx, level))
Danny Mayer's avatar
Danny Mayer committed
879 880 881 882 883 884 885 886 887
		return;

	va_start(ap, fmt);
	vsnprintf(msgbuf, sizeof(msgbuf), fmt, ap);
	va_end(ap);

	if (address == NULL) {
		isc_log_iwrite(isc_lctx, category, module, level,
			       msgcat, msgset, message,
888
			       "socket %p line %d: %s", sock, lineno, msgbuf);
889
	} else {
Mark Andrews's avatar
Mark Andrews committed
890
		isc_sockaddr_format(address, peerbuf, sizeof(peerbuf));
891 892
		isc_log_iwrite(isc_lctx, category, module, level,
			       msgcat, msgset, message,
893 894
				   "socket %p line %d peer %s: %s", sock, lineno,
				   peerbuf, msgbuf);
Danny Mayer's avatar
Danny Mayer committed
895
	}
896

Danny Mayer's avatar
Danny Mayer committed
897
}
898

Danny Mayer's avatar
Danny Mayer committed
899
/*
900
 * Make an fd SOCKET non-blocking.
Danny Mayer's avatar
Danny Mayer committed
901 902
 */
static isc_result_t
903
make_nonblock(SOCKET fd) {
Danny Mayer's avatar
Danny Mayer committed
904 905
	int ret;
	unsigned long flags = 1;
906
	char strbuf[ISC_STRERRORSIZE];
Danny Mayer's avatar
Danny Mayer committed
907 908

	/* Set the socket to non-blocking */
909
	ret = ioctlsocket(fd, FIONBIO, &flags);
Danny Mayer's avatar
Danny Mayer committed
910 911

	if (ret == -1) {
912
		isc__strerror(errno, strbuf, sizeof(strbuf));
Danny Mayer's avatar
Danny Mayer committed
913 914
		UNEXPECTED_ERROR(__FILE__, __LINE__,
				 "ioctlsocket(%d, FIOBIO, %d): %s",
915
				 fd, flags, strbuf);
Danny Mayer's avatar
Danny Mayer committed
916 917 918 919 920 921

		return (ISC_R_UNEXPECTED);
	}

	return (ISC_R_SUCCESS);
}
922

Danny Mayer's avatar
Danny Mayer committed
923
/*
924
 * Windows 2000 systems incorrectly cause UDP sockets using WSARecvFrom
925 926 927 928 929 930 931
 * to not work correctly, returning a WSACONNRESET error when a WSASendTo
 * fails with an "ICMP port unreachable" response and preventing the
 * socket from using the WSARecvFrom in subsequent operations.
 * The function below fixes this, but requires that Windows 2000
 * Service Pack 2 or later be installed on the system.  NT 4.0
 * systems are not affected by this and work correctly.
 * See Microsoft Knowledge Base Article Q263823 for details of this.
Danny Mayer's avatar
Danny Mayer committed
932
 */
933 934 935 936 937 938
isc_result_t
connection_reset_fix(SOCKET fd) {
	DWORD dwBytesReturned = 0;
	BOOL  bNewBehavior = FALSE;
	DWORD status;

939
	if (isc_win32os_versioncheck(5, 0, 0, 0) < 0)
940 941
		return (ISC_R_SUCCESS); /*  NT 4.0 has no problem */

942
	/* disable bad behavior using IOCTL: SIO_UDP_CONNRESET */
943 944 945 946 947
	status = WSAIoctl(fd, SIO_UDP_CONNRESET, &bNewBehavior,
			  sizeof(bNewBehavior), NULL, 0,
			  &dwBytesReturned, NULL, NULL);
	if (status != SOCKET_ERROR)
		return (ISC_R_SUCCESS);
948 949 950 951 952
	else {
		UNEXPECTED_ERROR(__FILE__, __LINE__,
				 "WSAIoctl(SIO_UDP_CONNRESET, oldBehaviour) %s",
				 isc_msgcat_get(isc_msgcat, ISC_MSGSET_GENERAL,
						ISC_MSG_FAILED, "failed"));
953
		return (ISC_R_UNEXPECTED);
954
	}
Danny Mayer's avatar
Danny Mayer committed
955 956 957 958 959 960 961 962 963 964 965 966 967
}

/*
 * Construct an iov array and attach it to the msghdr passed in.  This is
 * the SEND constructor, which will use the used region of the buffer
 * (if using a buffer list) or will use the internal region (if a single
 * buffer I/O is requested).
 *
 * Nothing can be NULL, and the done event must list at least one buffer
 * on the buffer linked list for this function to be meaningful.
 */
static void
build_msghdr_send(isc_socket_t *sock, isc_socketevent_t *dev,
968 969
		  struct msghdr *msg, char *cmsg, WSABUF *iov,
		  IoCompletionInfo  *lpo)
970
{
Danny Mayer's avatar
Danny Mayer committed
971 972
	unsigned int iovcount;
	isc_buffer_t *buffer;
973
	buflist_t  *cpbuffer;
Danny Mayer's avatar
Danny Mayer committed
974 975 976 977
	isc_region_t used;
	size_t write_count;
	size_t skip_count;

Mark Andrews's avatar
Mark Andrews committed
978
	memset(msg, 0, sizeof(*msg));
Danny Mayer's avatar
Danny Mayer committed
979

980
	memmove(&msg->to_addr, &dev->address.type, dev->address.length);
981
	msg->to_addr_len = dev->address.length;
Danny Mayer's avatar
Danny Mayer committed
982 983 984 985 986 987 988 989 990 991

	buffer = ISC_LIST_HEAD(dev->bufferlist);
	write_count = 0;
	iovcount = 0;

	/*
	 * Single buffer I/O?  Skip what we've done so far in this region.
	 */
	if (buffer == NULL) {
		write_count = dev->region.length - dev->n;
992 993 994 995 996 997 998 999 1000 1001
		cpbuffer = HeapAlloc(hHeapHandle, HEAP_ZERO_MEMORY, sizeof(buflist_t));
		RUNTIME_CHECK(cpbuffer != NULL);
		cpbuffer->buf = HeapAlloc(hHeapHandle, HEAP_ZERO_MEMORY, write_count);
		RUNTIME_CHECK(cpbuffer->buf != NULL);

		socket_log(__LINE__, sock, NULL, TRACE,
		   isc_msgcat, ISC_MSGSET_SOCKET, ISC_MSG_ACCEPTLOCK,
		   "alloc_buffer %p %d %p %d", cpbuffer, sizeof(buflist_t),
		   cpbuffer->buf, write_count);

1002
		memmove(cpbuffer->buf,(dev->region.base + dev->n), write_count);
1003
		cpbuffer->buflen = (unsigned int)write_count;
1004 1005
		ISC_LIST_ENQUEUE(lpo->bufferlist, cpbuffer, link);
		iov[0].buf = cpbuffer->buf;
1006
		iov[0].len = (u_long)write_count;
Danny Mayer's avatar
Danny Mayer committed
1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030
		iovcount = 1;

		goto config;
	}

	/*
	 * Multibuffer I/O.
	 * Skip the data in the buffer list that we have already written.
	 */
	skip_count = dev->n;
	while (buffer != NULL) {
		REQUIRE(ISC_BUFFER_VALID(buffer));
		if (skip_count < isc_buffer_usedlength(buffer))
			break;
		skip_count -= isc_buffer_usedlength(buffer);
		buffer = ISC_LIST_NEXT(buffer, link);
	}

	while (buffer != NULL) {
		INSIST(iovcount < MAXSCATTERGATHER_SEND);

		isc_buffer_usedregion(buffer, &used);

		if (used.length > 0) {
1031
			int uselen = (int)(used.length - skip_count);
1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
			cpbuffer = HeapAlloc(hHeapHandle, HEAP_ZERO_MEMORY, sizeof(buflist_t));
			RUNTIME_CHECK(cpbuffer != NULL);
			cpbuffer->buf = HeapAlloc(hHeapHandle, HEAP_ZERO_MEMORY, uselen);
			RUNTIME_CHECK(cpbuffer->buf != NULL);

			socket_log(__LINE__, sock, NULL, TRACE,
			   isc_msgcat, ISC_MSGSET_SOCKET, ISC_MSG_ACCEPTLOCK,
			   "alloc_buffer %p %d %p %d", cpbuffer, sizeof(buflist_t),
			   cpbuffer->buf, write_count);

1042
			memmove(cpbuffer->buf,(used.base + skip_count), uselen);
1043 1044
			cpbuffer->buflen = uselen;
			iov[iovcount].buf = cpbuffer->buf;
1045
			iov[iovcount].len = (u_long)(used.length - skip_count);
1046
			write_count += uselen;
Danny Mayer's avatar
Danny Mayer committed
1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057
			skip_count = 0;
			iovcount++;
		}
		buffer = ISC_LIST_NEXT(buffer, link);
	}

	INSIST(skip_count == 0);

 config:
	msg->msg_iov = iov;
	msg->msg_iovlen = iovcount;
1058
	msg->msg_totallen = (u_int)write_count;
Danny Mayer's avatar
Danny Mayer committed
1059 1060 1061 1062
}

static void
set_dev_address(isc_sockaddr_t *address, isc_socket_t *sock,