mem.c 41.9 KB
Newer Older
Bob Halley's avatar
base  
Bob Halley committed
1
/*
Mark Andrews's avatar
Mark Andrews committed
2
 * Copyright (C) 2004, 2005  Internet Systems Consortium, Inc. ("ISC")
Mark Andrews's avatar
Mark Andrews committed
3
 * Copyright (C) 1997-2003  Internet Software Consortium.
4
 *
Bob Halley's avatar
base  
Bob Halley committed
5 6 7
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
8
 *
Mark Andrews's avatar
Mark Andrews committed
9 10 11 12 13 14 15
 * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
 * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
 * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
Bob Halley's avatar
base  
Bob Halley committed
16 17
 */

18
/* $Id: mem.c,v 1.121 2005/06/10 07:00:19 marka Exp $ */
19 20

/*! \file */
David Lawrence's avatar
David Lawrence committed
21

Bob Halley's avatar
Bob Halley committed
22
#include <config.h>
Bob Halley's avatar
base  
Bob Halley committed
23 24 25

#include <stdio.h>
#include <stdlib.h>
Bob Halley's avatar
Bob Halley committed
26
#include <stddef.h>
Bob Halley's avatar
base  
Bob Halley committed
27

Michael Graff's avatar
Michael Graff committed
28 29
#include <limits.h>

30
#include <isc/magic.h>
Bob Halley's avatar
Bob Halley committed
31
#include <isc/mem.h>
32
#include <isc/msgs.h>
33
#include <isc/ondestroy.h>
34
#include <isc/string.h>
Bob Halley's avatar
Bob Halley committed
35

Bob Halley's avatar
update  
Bob Halley committed
36
#include <isc/mutex.h>
Michael Graff's avatar
Michael Graff committed
37
#include <isc/util.h>
Bob Halley's avatar
update  
Bob Halley committed
38

39 40 41
#define MCTXLOCK(m, l) if (((m)->flags & ISC_MEMFLAG_NOLOCK) == 0) LOCK(l)
#define MCTXUNLOCK(m, l) if (((m)->flags & ISC_MEMFLAG_NOLOCK) == 0) UNLOCK(l)

42 43 44
#ifndef ISC_MEM_DEBUGGING
#define ISC_MEM_DEBUGGING 0
#endif
45
LIBISC_EXTERNAL_DATA unsigned int isc_mem_debugging = ISC_MEM_DEBUGGING;
46

47
/*!
48 49 50 51
 * Define ISC_MEM_USE_INTERNAL_MALLOC=1 to use the internal malloc()
 * implementation in preference to the system one.  The internal malloc()
 * is very space-efficient, and quite fast on uniprocessor systems.  It
 * performs poorly on multiprocessor machines.
52 53
 * JT: we can overcome the performance issue on multiprocessor machines
 * by carefully separating memory contexts.
54 55
 */
#ifndef ISC_MEM_USE_INTERNAL_MALLOC
56
#define ISC_MEM_USE_INTERNAL_MALLOC 1
57 58
#endif

59 60 61 62 63 64
/*
 * Constants.
 */

#define DEF_MAX_SIZE		1100
#define DEF_MEM_TARGET		4096
65 66
#define ALIGNMENT_SIZE		8		/*%< must be a power of 2 */
#define NUM_BASIC_BLOCKS	64		/*%< must be > 1 */
67
#define TABLE_INCREMENT		1024
Michael Graff's avatar
Michael Graff committed
68
#define DEBUGLIST_COUNT		1024
69

Bob Halley's avatar
base  
Bob Halley committed
70 71 72
/*
 * Types.
 */
73
#if ISC_MEM_TRACKLINES
Michael Graff's avatar
Michael Graff committed
74 75 76 77
typedef struct debuglink debuglink_t;
struct debuglink {
	ISC_LINK(debuglink_t)	link;
	const void	       *ptr[DEBUGLIST_COUNT];
78
	unsigned int		size[DEBUGLIST_COUNT];
Michael Graff's avatar
Michael Graff committed
79 80 81 82
	const char	       *file[DEBUGLIST_COUNT];
	unsigned int		line[DEBUGLIST_COUNT];
	unsigned int		count;
};
Bob Halley's avatar
base  
Bob Halley committed
83

Michael Graff's avatar
Michael Graff committed
84 85 86 87 88 89
#define FLARG_PASS	, file, line
#define FLARG		, const char *file, int line
#else
#define FLARG_PASS
#define FLARG
#endif
90

Michael Graff's avatar
Michael Graff committed
91
typedef struct element element;
92 93 94
struct element {
	element *		next;
};
Bob Halley's avatar
base  
Bob Halley committed
95 96

typedef struct {
97
	/*!
Bob Halley's avatar
base  
Bob Halley committed
98 99
	 * This structure must be ALIGNMENT_SIZE bytes.
	 */
100 101
	union {
		size_t		size;
102
		isc_mem_t	*ctx;
103 104
		char		bytes[ALIGNMENT_SIZE];
	} u;
Bob Halley's avatar
Bob Halley committed
105
} size_info;
Bob Halley's avatar
base  
Bob Halley committed
106 107

struct stats {
Bob Halley's avatar
Bob Halley committed
108 109
	unsigned long		gets;
	unsigned long		totalgets;
110
#if ISC_MEM_USE_INTERNAL_MALLOC
Bob Halley's avatar
Bob Halley committed
111 112
	unsigned long		blocks;
	unsigned long		freefrags;
113
#endif /* ISC_MEM_USE_INTERNAL_MALLOC */
Bob Halley's avatar
base  
Bob Halley committed
114 115
};

116 117
#define MEM_MAGIC		ISC_MAGIC('M', 'e', 'm', 'C')
#define VALID_CONTEXT(c)	ISC_MAGIC_VALID(c, MEM_MAGIC)
Bob Halley's avatar
Bob Halley committed
118

119
#if ISC_MEM_TRACKLINES
120
typedef ISC_LIST(debuglink_t)	debuglist_t;
121
#endif
122

123
struct isc_mem {
Bob Halley's avatar
Bob Halley committed
124
	unsigned int		magic;
125
	isc_ondestroy_t		ondestroy;
126
	unsigned int		flags;
Bob Halley's avatar
Bob Halley committed
127
	isc_mutex_t		lock;
128 129 130
	isc_memalloc_t		memalloc;
	isc_memfree_t		memfree;
	void *			arg;
Bob Halley's avatar
base  
Bob Halley committed
131
	size_t			max_size;
132
	isc_boolean_t		checkfree;
Bob Halley's avatar
base  
Bob Halley committed
133
	struct stats *		stats;
134
	unsigned int		references;
135
	size_t			quota;
Bob Halley's avatar
Bob Halley committed
136
	size_t			total;
137
	size_t			inuse;
138
	size_t			maxinuse;
139 140 141 142 143
	size_t			hi_water;
	size_t			lo_water;
	isc_boolean_t		hi_called;
	isc_mem_water_t		water;
	void *			water_arg;
Michael Graff's avatar
Michael Graff committed
144
	ISC_LIST(isc_mempool_t)	pools;
145 146 147 148 149 150 151 152 153 154 155 156

#if ISC_MEM_USE_INTERNAL_MALLOC
	size_t			mem_target;
	element **		freelists;
	element *		basic_blocks;
	unsigned char **	basic_table;
	unsigned int		basic_table_count;
	unsigned int		basic_table_size;
	unsigned char *		lowest;
	unsigned char *		highest;
#endif /* ISC_MEM_USE_INTERNAL_MALLOC */

157
#if ISC_MEM_TRACKLINES
158
	debuglist_t *	 	debuglist;
Michael Graff's avatar
Michael Graff committed
159
#endif
160 161

	unsigned int		memalloc_failures;
Michael Graff's avatar
Michael Graff committed
162 163
};

164 165
#define MEMPOOL_MAGIC		ISC_MAGIC('M', 'E', 'M', 'p')
#define VALID_MEMPOOL(c)	ISC_MAGIC_VALID(c, MEMPOOL_MAGIC)
Michael Graff's avatar
Michael Graff committed
166 167

struct isc_mempool {
Michael Graff's avatar
Michael Graff committed
168
	/* always unlocked */
169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184
	unsigned int	magic;		/*%< magic number */
	isc_mutex_t    *lock;		/*%< optional lock */
	isc_mem_t      *mctx;		/*%< our memory context */
	/*%< locked via the memory context's lock */
	ISC_LINK(isc_mempool_t)	link;	/*%< next pool in this mem context */
	/*%< optionally locked from here down */
	element	       *items;		/*%< low water item list */
	size_t		size;		/*%< size of each item on this pool */
	unsigned int	maxalloc;	/*%< max number of items allowed */
	unsigned int	allocated;	/*%< # of items currently given out */
	unsigned int	freecount;	/*%< # of items on reserved list */
	unsigned int	freemax;	/*%< # of items allowed on free list */
	unsigned int	fillcount;	/*%< # of items to fetch on each fill */
	/*%< Stats only. */
	unsigned int	gets;		/*%< # of requests to this pool */
	/*%< Debugging only. */
185
#if ISC_MEMPOOL_NAMES
186
	char		name[16];	/*%< printed name in stats reports */
Michael Graff's avatar
Michael Graff committed
187
#endif
Bob Halley's avatar
base  
Bob Halley committed
188 189
};

190 191 192
/*
 * Private Inline-able.
 */
Bob Halley's avatar
base  
Bob Halley committed
193

194
#if ! ISC_MEM_TRACKLINES
Michael Graff's avatar
Michael Graff committed
195 196 197
#define ADD_TRACE(a, b, c, d, e)
#define DELETE_TRACE(a, b, c, d, e)
#else
198
#define ADD_TRACE(a, b, c, d, e) \
199 200 201 202 203 204
	do { \
		if ((isc_mem_debugging & (ISC_MEM_DEBUGTRACE | \
					  ISC_MEM_DEBUGRECORD)) != 0 && \
		     b != NULL) \
		         add_trace_entry(a, b, c, d, e); \
	} while (0)
Michael Graff's avatar
Michael Graff committed
205 206
#define DELETE_TRACE(a, b, c, d, e)	delete_trace_entry(a, b, c, d, e)

207 208 209
static void
print_active(isc_mem_t *ctx, FILE *out);

210
/*!
Michael Graff's avatar
Michael Graff committed
211 212 213 214
 * mctx must be locked.
 */
static inline void
add_trace_entry(isc_mem_t *mctx, const void *ptr, unsigned int size
Michael Graff's avatar
pasto  
Michael Graff committed
215
		FLARG)
Michael Graff's avatar
Michael Graff committed
216 217 218 219
{
	debuglink_t *dl;
	unsigned int i;

220
	if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0)
221 222 223 224
		fprintf(stderr, isc_msgcat_get(isc_msgcat, ISC_MSGSET_MEM,
					       ISC_MSG_ADDTRACE,
					       "add %p size %u "
					       "file %s line %u mctx %p\n"),
225
			ptr, size, file, line, mctx);
Michael Graff's avatar
Michael Graff committed
226

227
	if (mctx->debuglist == NULL)
Michael Graff's avatar
Michael Graff committed
228 229
		return;

230 231 232 233
	if (size > mctx->max_size)
		size = mctx->max_size;

	dl = ISC_LIST_HEAD(mctx->debuglist[size]);
Michael Graff's avatar
Michael Graff committed
234 235 236
	while (dl != NULL) {
		if (dl->count == DEBUGLIST_COUNT)
			goto next;
237
		for (i = 0; i < DEBUGLIST_COUNT; i++) {
Michael Graff's avatar
Michael Graff committed
238 239
			if (dl->ptr[i] == NULL) {
				dl->ptr[i] = ptr;
240
				dl->size[i] = size;
Michael Graff's avatar
Michael Graff committed
241 242 243 244 245 246 247 248 249 250 251 252 253 254
				dl->file[i] = file;
				dl->line[i] = line;
				dl->count++;
				return;
			}
		}
	next:
		dl = ISC_LIST_NEXT(dl, link);
	}

	dl = malloc(sizeof(debuglink_t));
	INSIST(dl != NULL);

	ISC_LINK_INIT(dl, link);
255
	for (i = 1; i < DEBUGLIST_COUNT; i++) {
Michael Graff's avatar
Michael Graff committed
256
		dl->ptr[i] = NULL;
257
		dl->size[i] = 0;
Michael Graff's avatar
Michael Graff committed
258 259 260 261 262
		dl->file[i] = NULL;
		dl->line[i] = 0;
	}

	dl->ptr[0] = ptr;
263
	dl->size[0] = size;
Michael Graff's avatar
Michael Graff committed
264 265 266 267
	dl->file[0] = file;
	dl->line[0] = line;
	dl->count = 1;

268
	ISC_LIST_PREPEND(mctx->debuglist[size], dl, link);
Michael Graff's avatar
Michael Graff committed
269 270 271 272 273 274 275 276 277
}

static inline void
delete_trace_entry(isc_mem_t *mctx, const void *ptr, unsigned int size,
		   const char *file, unsigned int line)
{
	debuglink_t *dl;
	unsigned int i;

278
	if ((isc_mem_debugging & ISC_MEM_DEBUGTRACE) != 0)
279 280 281 282
		fprintf(stderr, isc_msgcat_get(isc_msgcat, ISC_MSGSET_MEM,
					       ISC_MSG_DELTRACE,
					       "del %p size %u "
					       "file %s line %u mctx %p\n"),
283
			ptr, size, file, line, mctx);
Michael Graff's avatar
Michael Graff committed
284

285
	if (mctx->debuglist == NULL)
Michael Graff's avatar
Michael Graff committed
286 287
		return;

288 289 290 291
	if (size > mctx->max_size)
		size = mctx->max_size;

	dl = ISC_LIST_HEAD(mctx->debuglist[size]);
Michael Graff's avatar
Michael Graff committed
292
	while (dl != NULL) {
293
		for (i = 0; i < DEBUGLIST_COUNT; i++) {
Michael Graff's avatar
Michael Graff committed
294 295
			if (dl->ptr[i] == ptr) {
				dl->ptr[i] = NULL;
296
				dl->size[i] = 0;
Michael Graff's avatar
Michael Graff committed
297 298 299 300 301 302
				dl->file[i] = NULL;
				dl->line[i] = 0;

				INSIST(dl->count > 0);
				dl->count--;
				if (dl->count == 0) {
303
					ISC_LIST_UNLINK(mctx->debuglist[size],
Michael Graff's avatar
Michael Graff committed
304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
							dl, link);
					free(dl);
				}
				return;
			}
		}
		dl = ISC_LIST_NEXT(dl, link);
	}

	/*
	 * If we get here, we didn't find the item on the list.  We're
	 * screwed.
	 */
	INSIST(dl != NULL);
}
#endif /* ISC_MEM_TRACKLINES */

321
#if ISC_MEM_USE_INTERNAL_MALLOC
322 323 324 325 326
static inline size_t
rmsize(size_t size) {
	/*
 	 * round down to ALIGNMENT_SIZE
	 */
327
	return (size & (~(ALIGNMENT_SIZE - 1)));
328 329
}

330
static inline size_t
Bob Halley's avatar
base  
Bob Halley committed
331
quantize(size_t size) {
332
	/*!
333
	 * Round up the result in order to get a size big
Bob Halley's avatar
base  
Bob Halley committed
334 335 336
	 * enough to satisfy the request and be aligned on ALIGNMENT_SIZE
	 * byte boundaries.
	 */
337

338 339
	if (size == 0)
		return (ALIGNMENT_SIZE);
340
	return ((size + ALIGNMENT_SIZE - 1) & (~(ALIGNMENT_SIZE - 1)));
Bob Halley's avatar
base  
Bob Halley committed
341 342
}

343
static inline isc_boolean_t
344
more_basic_blocks(isc_mem_t *ctx) {
345 346 347 348 349
	void *new;
	unsigned char *curr, *next;
	unsigned char *first, *last;
	unsigned char **table;
	unsigned int table_size;
Bob Halley's avatar
Bob Halley committed
350
	size_t increment;
351 352 353 354
	int i;

	/* Require: we hold the context lock. */

355 356 357
	/*
	 * Did we hit the quota for this context?
	 */
Bob Halley's avatar
Bob Halley committed
358 359
	increment = NUM_BASIC_BLOCKS * ctx->mem_target;
	if (ctx->quota != 0 && ctx->total + increment > ctx->quota)
360
		return (ISC_FALSE);
361 362 363

	INSIST(ctx->basic_table_count <= ctx->basic_table_size);
	if (ctx->basic_table_count == ctx->basic_table_size) {
364
		table_size = ctx->basic_table_size + TABLE_INCREMENT;
365
		table = (ctx->memalloc)(ctx->arg,
Andreas Gustafsson's avatar
Andreas Gustafsson committed
366
					table_size * sizeof(unsigned char *));
367 368
		if (table == NULL) {
			ctx->memalloc_failures++;
369
			return (ISC_FALSE);
370
		}
371 372 373
		if (ctx->basic_table_size != 0) {
			memcpy(table, ctx->basic_table,
			       ctx->basic_table_size *
Andreas Gustafsson's avatar
Andreas Gustafsson committed
374
			       sizeof(unsigned char *));
375
			(ctx->memfree)(ctx->arg, ctx->basic_table);
376
		}
377 378
		ctx->basic_table = table;
		ctx->basic_table_size = table_size;
379
	}
380

381
	new = (ctx->memalloc)(ctx->arg, NUM_BASIC_BLOCKS * ctx->mem_target);
382 383
	if (new == NULL) {
		ctx->memalloc_failures++;
384
		return (ISC_FALSE);
385
	}
Bob Halley's avatar
Bob Halley committed
386
	ctx->total += increment;
387 388
	ctx->basic_table[ctx->basic_table_count] = new;
	ctx->basic_table_count++;
389

390 391 392
	curr = new;
	next = curr + ctx->mem_target;
	for (i = 0; i < (NUM_BASIC_BLOCKS - 1); i++) {
393
		((element *)curr)->next = (element *)next;
394 395 396 397 398 399 400
		curr = next;
		next += ctx->mem_target;
	}
	/*
	 * curr is now pointing at the last block in the
	 * array.
	 */
Bob Halley's avatar
Bob Halley committed
401
	((element *)curr)->next = NULL;
402 403 404 405 406 407 408
	first = new;
	last = first + NUM_BASIC_BLOCKS * ctx->mem_target - 1;
	if (first < ctx->lowest || ctx->lowest == NULL)
		ctx->lowest = first;
	if (last > ctx->highest)
		ctx->highest = last;
	ctx->basic_blocks = new;
409 410

	return (ISC_TRUE);
411 412
}

413 414 415 416 417 418 419
static inline isc_boolean_t
more_frags(isc_mem_t *ctx, size_t new_size) {
	int i, frags;
	size_t total_size;
	void *new;
	unsigned char *curr, *next;

420
	/*!
421 422 423 424 425 426 427 428 429 430
	 * Try to get more fragments by chopping up a basic block.
	 */

	if (ctx->basic_blocks == NULL) {
		if (!more_basic_blocks(ctx)) {
			/*
			 * We can't get more memory from the OS, or we've
			 * hit the quota for this context.
			 */
			/*
431
			 * XXXRTH  "At quota" notification here.
432
			 */
433
			return (ISC_FALSE);
434 435 436 437 438 439 440 441 442 443 444 445 446 447 448
		}
	}

	total_size = ctx->mem_target;
	new = ctx->basic_blocks;
	ctx->basic_blocks = ctx->basic_blocks->next;
	frags = total_size / new_size;
	ctx->stats[new_size].blocks++;
	ctx->stats[new_size].freefrags += frags;
	/*
	 * Set up a linked-list of blocks of size
	 * "new_size".
	 */
	curr = new;
	next = curr + new_size;
449
	total_size -= new_size;
450 451 452 453
	for (i = 0; i < (frags - 1); i++) {
		((element *)curr)->next = (element *)next;
		curr = next;
		next += new_size;
454 455 456 457 458 459 460 461 462 463
		total_size -= new_size;
	}
	/*
	 * Add the remaining fragment of the basic block to a free list.
	 */
	total_size = rmsize(total_size);
	if (total_size > 0) {
		((element *)next)->next = ctx->freelists[total_size];
		ctx->freelists[total_size] = (element *)next;
		ctx->stats[total_size].freefrags++;
464 465 466 467 468 469 470 471 472 473 474
	}
	/*
	 * curr is now pointing at the last block in the
	 * array.
	 */
	((element *)curr)->next = NULL;
	ctx->freelists[new_size] = new;

	return (ISC_TRUE);
}

Michael Graff's avatar
Michael Graff committed
475
static inline void *
476
mem_getunlocked(isc_mem_t *ctx, size_t size) {
Michael Graff's avatar
Michael Graff committed
477 478
	size_t new_size = quantize(size);
	void *ret;
Bob Halley's avatar
base  
Bob Halley committed
479 480

	if (size >= ctx->max_size || new_size >= ctx->max_size) {
481 482 483
		/*
		 * memget() was called on something beyond our upper limit.
		 */
Bob Halley's avatar
Bob Halley committed
484 485 486 487
		if (ctx->quota != 0 && ctx->total + size > ctx->quota) {
			ret = NULL;
			goto done;
		}
Mark Andrews's avatar
Mark Andrews committed
488
		ret = (ctx->memalloc)(ctx->arg, size);
489 490 491
		if (ret == NULL) {
			ctx->memalloc_failures++;
			goto done;
Bob Halley's avatar
base  
Bob Halley committed
492
		}
493 494 495 496 497 498 499 500 501 502
		ctx->total += size;
		ctx->inuse += size;
		ctx->stats[ctx->max_size].gets++;
		ctx->stats[ctx->max_size].totalgets++;
		/*
		 * If we don't set new_size to size, then the
		 * ISC_MEM_FILL code might write over bytes we
		 * don't own.
		 */
		new_size = size;
503
		goto done;
Bob Halley's avatar
base  
Bob Halley committed
504 505
	}

506
	/*
Bob Halley's avatar
base  
Bob Halley committed
507 508 509 510
	 * If there are no blocks in the free list for this size, get a chunk
	 * of memory and then break it up into "new_size"-sized blocks, adding
	 * them to the free list.
	 */
511 512
	if (ctx->freelists[new_size] == NULL && !more_frags(ctx, new_size))
		return (NULL);
Bob Halley's avatar
base  
Bob Halley committed
513

514 515 516
	/*
	 * The free list uses the "rounded-up" size "new_size".
	 */
Bob Halley's avatar
base  
Bob Halley committed
517 518 519
	ret = ctx->freelists[new_size];
	ctx->freelists[new_size] = ctx->freelists[new_size]->next;

520
	/*
Bob Halley's avatar
base  
Bob Halley committed
521 522 523 524 525 526 527 528
	 * The stats[] uses the _actual_ "size" requested by the
	 * caller, with the caveat (in the code above) that "size" >= the
	 * max. size (max_size) ends up getting recorded as a call to
	 * max_size.
	 */
	ctx->stats[size].gets++;
	ctx->stats[size].totalgets++;
	ctx->stats[new_size].freefrags--;
529
	ctx->inuse += new_size;
Bob Halley's avatar
base  
Bob Halley committed
530 531 532

 done:

533
#if ISC_MEM_FILL
534 535
	if (ret != NULL)
		memset(ret, 0xbe, new_size); /* Mnemonic for "beef". */
David Lawrence's avatar
David Lawrence committed
536 537 538
#endif

	return (ret);
Michael Graff's avatar
Michael Graff committed
539 540
}

541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
#if ISC_MEM_FILL && ISC_MEM_CHECKOVERRUN
static inline void
check_overrun(void *mem, size_t size, size_t new_size) {
	unsigned char *cp;

	cp = (unsigned char *)mem;
	cp += size;
	while (size < new_size) {
		INSIST(*cp == 0xbe);
		cp++;
		size++;
	}
}
#endif

Michael Graff's avatar
Michael Graff committed
556
static inline void
557
mem_putunlocked(isc_mem_t *ctx, void *mem, size_t size) {
Michael Graff's avatar
Michael Graff committed
558
	size_t new_size = quantize(size);
Bob Halley's avatar
base  
Bob Halley committed
559

560
	if (size == ctx->max_size || new_size >= ctx->max_size) {
561 562 563
		/*
		 * memput() called on something beyond our upper limit.
		 */
564
#if ISC_MEM_FILL
Mark Andrews's avatar
Mark Andrews committed
565 566
		memset(mem, 0xde, size); /* Mnemonic for "dead". */
#endif
567
		(ctx->memfree)(ctx->arg, mem);
Bob Halley's avatar
base  
Bob Halley committed
568 569
		INSIST(ctx->stats[ctx->max_size].gets != 0);
		ctx->stats[ctx->max_size].gets--;
Bob Halley's avatar
Bob Halley committed
570
		INSIST(size <= ctx->total);
571
		ctx->inuse -= size;
Bob Halley's avatar
Bob Halley committed
572
		ctx->total -= size;
Michael Graff's avatar
Michael Graff committed
573
		return;
Bob Halley's avatar
base  
Bob Halley committed
574 575
	}

576 577
#if ISC_MEM_FILL
#if ISC_MEM_CHECKOVERRUN
578 579
	check_overrun(mem, size, new_size);
#endif
Mark Andrews's avatar
Mark Andrews committed
580 581
	memset(mem, 0xde, new_size); /* Mnemonic for "dead". */
#endif
582

583 584 585
	/*
	 * The free list uses the "rounded-up" size "new_size".
	 */
Mark Andrews's avatar
Mark Andrews committed
586 587
	((element *)mem)->next = ctx->freelists[new_size];
	ctx->freelists[new_size] = (element *)mem;
Bob Halley's avatar
base  
Bob Halley committed
588

589
	/*
Bob Halley's avatar
base  
Bob Halley committed
590 591 592 593 594 595 596 597
	 * The stats[] uses the _actual_ "size" requested by the
	 * caller, with the caveat (in the code above) that "size" >= the
	 * max. size (max_size) ends up getting recorded as a call to
	 * max_size.
	 */
	INSIST(ctx->stats[size].gets != 0);
	ctx->stats[size].gets--;
	ctx->stats[new_size].freefrags++;
598
	ctx->inuse -= new_size;
Bob Halley's avatar
base  
Bob Halley committed
599 600
}

601 602
#else /* ISC_MEM_USE_INTERNAL_MALLOC */

603
/*!
604 605 606 607 608 609 610 611 612 613 614
 * Perform a malloc, doing memory filling and overrun detection as necessary.
 */
static inline void *
mem_get(isc_mem_t *ctx, size_t size) {
	char *ret;

#if ISC_MEM_CHECKOVERRUN
	size += 1;
#endif

	ret = (ctx->memalloc)(ctx->arg, size);
615 616
	if (ret == NULL)
		ctx->memalloc_failures++;	
617 618 619 620 621 622 623 624 625 626 627 628 629 630

#if ISC_MEM_FILL
	if (ret != NULL)
		memset(ret, 0xbe, size); /* Mnemonic for "beef". */
#else
#  if ISC_MEM_CHECKOVERRUN
	if (ret != NULL)
		ret[size-1] = 0xbe;
#  endif
#endif

	return (ret);
}

631
/*!
632 633 634 635 636
 * Perform a free, doing memory filling and overrun detection as necessary.
 */
static inline void
mem_put(isc_mem_t *ctx, void *mem, size_t size) {
#if ISC_MEM_CHECKOVERRUN
637
	INSIST(((unsigned char *)mem)[size] == 0xbe);
638 639 640
#endif
#if ISC_MEM_FILL
	memset(mem, 0xde, size); /* Mnemonic for "dead". */
Andreas Gustafsson's avatar
Andreas Gustafsson committed
641 642
#else
	UNUSED(size);
643 644 645 646
#endif
	(ctx->memfree)(ctx->arg, mem);
}

647
/*!
648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663
 * Update internal counters after a memory get.
 */
static inline void
mem_getstats(isc_mem_t *ctx, size_t size) {
	ctx->total += size;
	ctx->inuse += size;

	if (size > ctx->max_size) {
		ctx->stats[ctx->max_size].gets++;
		ctx->stats[ctx->max_size].totalgets++;
	} else {
		ctx->stats[size].gets++;
		ctx->stats[size].totalgets++;
	}
}

664
/*!
665 666 667 668 669 670 671 672 673 674
 * Update internal counters after a memory put.
 */
static inline void
mem_putstats(isc_mem_t *ctx, void *ptr, size_t size) {
	UNUSED(ptr);

	INSIST(ctx->inuse >= size);
	ctx->inuse -= size;

	if (size > ctx->max_size) {
675
		INSIST(ctx->stats[ctx->max_size].gets > 0U);
676 677
		ctx->stats[ctx->max_size].gets--;
	} else {
678
		INSIST(ctx->stats[size].gets > 0U);
679 680 681 682 683 684
		ctx->stats[size].gets--;
	}
}

#endif /* ISC_MEM_USE_INTERNAL_MALLOC */

David Lawrence's avatar
David Lawrence committed
685 686 687 688 689 690 691
/*
 * Private.
 */

static void *
default_memalloc(void *arg, size_t size) {
	UNUSED(arg);
692
	if (size == 0U)
693
		size = 1;
David Lawrence's avatar
David Lawrence committed
694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710
	return (malloc(size));
}

static void
default_memfree(void *arg, void *ptr) {
	UNUSED(arg);
	free(ptr);
}

/*
 * Public.
 */

isc_result_t
isc_mem_createx(size_t init_max_size, size_t target_size,
		isc_memalloc_t memalloc, isc_memfree_t memfree, void *arg,
		isc_mem_t **ctxp)
711 712 713 714 715 716 717 718 719 720
{
	return (isc_mem_createx2(init_max_size, target_size, memalloc, memfree,
				 arg, ctxp, 0));
				 
}

isc_result_t
isc_mem_createx2(size_t init_max_size, size_t target_size,
		 isc_memalloc_t memalloc, isc_memfree_t memfree, void *arg,
		 isc_mem_t **ctxp, unsigned int flags)
David Lawrence's avatar
David Lawrence committed
721 722
{
	isc_mem_t *ctx;
723
	isc_result_t result;
David Lawrence's avatar
David Lawrence committed
724 725 726 727 728

	REQUIRE(ctxp != NULL && *ctxp == NULL);
	REQUIRE(memalloc != NULL);
	REQUIRE(memfree != NULL);

729 730
	INSIST((ALIGNMENT_SIZE & (ALIGNMENT_SIZE - 1)) == 0);

731 732 733 734
#if !ISC_MEM_USE_INTERNAL_MALLOC
	UNUSED(target_size);
#endif

Andreas Gustafsson's avatar
Andreas Gustafsson committed
735
	ctx = (memalloc)(arg, sizeof(*ctx));
David Lawrence's avatar
David Lawrence committed
736 737 738
	if (ctx == NULL)
		return (ISC_R_NOMEMORY);

739 740
	if ((flags & ISC_MEMFLAG_NOLOCK) == 0 &&
	    isc_mutex_init(&ctx->lock) != ISC_R_SUCCESS) {
741 742
		UNEXPECTED_ERROR(__FILE__, __LINE__,
				 "isc_mutex_init() %s",
743 744 745 746
				 isc_msgcat_get(isc_msgcat,
						ISC_MSGSET_GENERAL,
						ISC_MSG_FAILED,
						"failed"));
747 748 749 750
		(memfree)(arg, ctx);
		return (ISC_R_UNEXPECTED);
	}

751
	if (init_max_size == 0U)
David Lawrence's avatar
David Lawrence committed
752 753 754
		ctx->max_size = DEF_MAX_SIZE;
	else
		ctx->max_size = init_max_size;
755
	ctx->flags = flags;
756 757 758 759
	ctx->references = 1;
	ctx->quota = 0;
	ctx->total = 0;
	ctx->inuse = 0;
760
	ctx->maxinuse = 0;
761 762 763 764 765 766 767 768 769 770 771
	ctx->hi_water = 0;
	ctx->lo_water = 0;
	ctx->hi_called = ISC_FALSE;
	ctx->water = NULL;
	ctx->water_arg = NULL;
	ctx->magic = MEM_MAGIC;
	isc_ondestroy_init(&ctx->ondestroy);
	ctx->memalloc = memalloc;
	ctx->memfree = memfree;
	ctx->arg = arg;
	ctx->stats = NULL;
772
	ctx->checkfree = ISC_TRUE;
773
#if ISC_MEM_TRACKLINES
774
	ctx->debuglist = NULL;
775
#endif
776 777 778 779 780 781 782
	ISC_LIST_INIT(ctx->pools);

#if ISC_MEM_USE_INTERNAL_MALLOC
	ctx->freelists = NULL;
#endif /* ISC_MEM_USE_INTERNAL_MALLOC */

	ctx->stats = (memalloc)(arg,
Andreas Gustafsson's avatar
Andreas Gustafsson committed
783
				(ctx->max_size+1) * sizeof(struct stats));
784 785 786 787
	if (ctx->stats == NULL) {
		result = ISC_R_NOMEMORY;
		goto error;
	}
Andreas Gustafsson's avatar
Andreas Gustafsson committed
788
	memset(ctx->stats, 0, (ctx->max_size + 1) * sizeof(struct stats));
789 790

#if ISC_MEM_USE_INTERNAL_MALLOC
David Lawrence's avatar
David Lawrence committed
791 792 793 794
	if (target_size == 0)
		ctx->mem_target = DEF_MEM_TARGET;
	else
		ctx->mem_target = target_size;
Andreas Gustafsson's avatar
Andreas Gustafsson committed
795
	ctx->freelists = (memalloc)(arg, ctx->max_size * sizeof(element *));
David Lawrence's avatar
David Lawrence committed
796
	if (ctx->freelists == NULL) {
797 798
		result = ISC_R_NOMEMORY;
		goto error;
David Lawrence's avatar
David Lawrence committed
799 800
	}
	memset(ctx->freelists, 0,
Andreas Gustafsson's avatar
Andreas Gustafsson committed
801
	       ctx->max_size * sizeof(element *));
David Lawrence's avatar
David Lawrence committed
802 803 804 805 806 807
	ctx->basic_blocks = NULL;
	ctx->basic_table = NULL;
	ctx->basic_table_count = 0;
	ctx->basic_table_size = 0;
	ctx->lowest = NULL;
	ctx->highest = NULL;
808 809
#endif /* ISC_MEM_USE_INTERNAL_MALLOC */

810
#if ISC_MEM_TRACKLINES
811 812 813 814
	if ((isc_mem_debugging & ISC_MEM_DEBUGRECORD) != 0) {
		unsigned int i;

		ctx->debuglist = (memalloc)(arg,
Andreas Gustafsson's avatar
Andreas Gustafsson committed
815
				      (ctx->max_size+1) * sizeof(debuglist_t));
816 817 818 819 820 821 822
		if (ctx->debuglist == NULL) {
			result = ISC_R_NOMEMORY;
			goto error;
		}
		for (i = 0; i <= ctx->max_size; i++)
			ISC_LIST_INIT(ctx->debuglist[i]);
	}
Michael Graff's avatar
Michael Graff committed
823
#endif
David Lawrence's avatar
David Lawrence committed
824

825 826
	ctx->memalloc_failures = 0;

David Lawrence's avatar
David Lawrence committed
827 828
	*ctxp = ctx;
	return (ISC_R_SUCCESS);
829 830

  error:
831 832
	if (ctx != NULL) {
		if (ctx->stats != NULL)
833 834
			(memfree)(arg, ctx->stats);
#if ISC_MEM_USE_INTERNAL_MALLOC
835
		if (ctx->freelists != NULL)
836 837
			(memfree)(arg, ctx->freelists);
#endif /* ISC_MEM_USE_INTERNAL_MALLOC */
838
#if ISC_MEM_TRACKLINES
839
		if (ctx->debuglist != NULL)
840 841
			(ctx->memfree)(ctx->arg, ctx->debuglist);
#endif /* ISC_MEM_TRACKLINES */
842 843
		if ((ctx->flags & ISC_MEMFLAG_NOLOCK) == 0)
			DESTROYLOCK(&ctx->lock);
844 845 846 847
		(memfree)(arg, ctx);
	}

	return (result);
David Lawrence's avatar
David Lawrence committed
848 849 850 851 852 853
}

isc_result_t
isc_mem_create(size_t init_max_size, size_t target_size,
	       isc_mem_t **ctxp)
{
854 855 856 857 858 859 860 861 862 863 864 865
	return (isc_mem_createx2(init_max_size, target_size,
				 default_memalloc, default_memfree, NULL,
				 ctxp, 0));
}

isc_result_t
isc_mem_create2(size_t init_max_size, size_t target_size,
		isc_mem_t **ctxp, unsigned int flags)
{
	return (isc_mem_createx2(init_max_size, target_size,
				 default_memalloc, default_memfree, NULL,
				 ctxp, flags));
David Lawrence's avatar
David Lawrence committed
866 867 868 869 870 871 872 873 874
}

static void
destroy(isc_mem_t *ctx) {
	unsigned int i;
	isc_ondestroy_t ondest;

	ctx->magic = 0;

875
#if ISC_MEM_USE_INTERNAL_MALLOC
David Lawrence's avatar
David Lawrence committed
876
	INSIST(ISC_LIST_EMPTY(ctx->pools));
877 878
#endif /* ISC_MEM_USE_INTERNAL_MALLOC */

879
#if ISC_MEM_TRACKLINES
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897
	if (ctx->debuglist != NULL) {
		if (ctx->checkfree) {
			for (i = 0; i <= ctx->max_size; i++) {
				if (!ISC_LIST_EMPTY(ctx->debuglist[i]))
					print_active(ctx, stderr);
				INSIST(ISC_LIST_EMPTY(ctx->debuglist[i]));
			}
		} else {
			debuglink_t *dl;

			for (i = 0; i <= ctx->max_size; i++)
				for (dl = ISC_LIST_HEAD(ctx->debuglist[i]);
				     dl != NULL;
				     dl = ISC_LIST_HEAD(ctx->debuglist[i])) {
					ISC_LIST_UNLINK(ctx->debuglist[i],
						 	dl, link);
					free(dl);
				}
898
		}
899
		(ctx->memfree)(ctx->arg, ctx->debuglist);
900
	}
Michael Graff's avatar
Michael Graff committed
901
#endif
David Lawrence's avatar
David Lawrence committed
902 903 904
	INSIST(ctx->references == 0);

	if (ctx->checkfree) {
905 906
		for (i = 0; i <= ctx->max_size; i++) {
#if ISC_MEM_TRACKLINES
Mark Andrews's avatar
Mark Andrews committed
907
			if (ctx->stats[i].gets != 0U)
908 909
				print_active(ctx, stderr);
#endif
910
			INSIST(ctx->stats[i].gets == 0U);
911
		}
David Lawrence's avatar
David Lawrence committed
912 913
	}

914
	(ctx->memfree)(ctx->arg, ctx->stats);
David Lawrence's avatar
David Lawrence committed
915

916
#if ISC_MEM_USE_INTERNAL_MALLOC
David Lawrence's avatar
David Lawrence committed
917 918 919 920
	for (i = 0; i < ctx->basic_table_count; i++)
		(ctx->memfree)(ctx->arg, ctx->basic_table[i]);
	(ctx->memfree)(ctx->arg, ctx->freelists);
	(ctx->memfree)(ctx->arg, ctx->basic_table);
921
#endif /* ISC_MEM_USE_INTERNAL_MALLOC */
David Lawrence's avatar
David Lawrence committed
922 923 924

	ondest = ctx->ondestroy;

925 926
	if ((ctx->flags & ISC_MEMFLAG_NOLOCK) == 0)
		DESTROYLOCK(&ctx->lock);
David Lawrence's avatar
David Lawrence committed
927 928 929 930 931 932 933 934 935 936
	(ctx->memfree)(ctx->arg, ctx);

	isc_ondestroy_notify(&ondest, ctx);
}

void
isc_mem_attach(isc_mem_t *source, isc_mem_t **targetp) {
	REQUIRE(VALID_CONTEXT(source));
	REQUIRE(targetp != NULL && *targetp == NULL);

937
	MCTXLOCK(source, &source->lock);
David Lawrence's avatar
David Lawrence committed
938
	source->references++;
939
	MCTXUNLOCK(source, &source->lock);
David Lawrence's avatar
David Lawrence committed
940 941 942 943 944 945 946 947 948 949 950 951 952

	*targetp = source;
}

void
isc_mem_detach(isc_mem_t **ctxp) {
	isc_mem_t *ctx;
	isc_boolean_t want_destroy = ISC_FALSE;

	REQUIRE(ctxp != NULL);
	ctx = *ctxp;
	REQUIRE(VALID_CONTEXT(ctx));

953
	MCTXLOCK(ctx, &ctx->lock);
David Lawrence's avatar
David Lawrence committed
954 955 956 957
	INSIST(ctx->references > 0);
	ctx->references--;
	if (ctx->references == 0)
		want_destroy = ISC_TRUE;
958
	MCTXUNLOCK(ctx, &ctx->lock);
David Lawrence's avatar
David Lawrence committed
959 960 961 962 963 964 965

	if (want_destroy)
		destroy(ctx);

	*ctxp = NULL;
}

966 967 968 969 970 971 972 973 974 975 976 977 978 979
/*
 * isc_mem_putanddetach() is the equivalent of:
 *
 * mctx = NULL;
 * isc_mem_attach(ptr->mctx, &mctx);
 * isc_mem_detach(&ptr->mctx);
 * isc_mem_put(mctx, ptr, sizeof(*ptr);
 * isc_mem_detach(&mctx);
 */

void
isc__mem_putanddetach(isc_mem_t **ctxp, void *ptr, size_t size FLARG) {
	isc_mem_t *ctx;
	isc_boolean_t want_destroy = ISC_FALSE;
980 981
	size_info *si;
	size_t oldsize;
982 983 984 985 986 987 988 989 990 991 992 993

	REQUIRE(ctxp != NULL);
	ctx = *ctxp;
	REQUIRE(VALID_CONTEXT(ctx));
	REQUIRE