opensslrsa_link.c 31.9 KB
Newer Older
1
/*
Tinderbox User's avatar
Tinderbox User committed
2
 * Copyright (C) 2004-2009, 2011, 2012  Internet Systems Consortium, Inc. ("ISC")
Mark Andrews's avatar
Mark Andrews committed
3
 * Copyright (C) 2000-2003  Internet Software Consortium.
4
 *
Automatic Updater's avatar
Automatic Updater committed
5
 * Permission to use, copy, modify, and/or distribute this software for any
6
7
8
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
Mark Andrews's avatar
Mark Andrews committed
9
10
11
12
13
14
15
 * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
 * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
 * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
16
17
18
19
 */

/*
 * Principal Author: Brian Wellington
Mark Andrews's avatar
Mark Andrews committed
20
 * $Id$
21
 */
22
#ifdef OPENSSL
23
24
#include <config.h>

Francis Dupont's avatar
Francis Dupont committed
25
#ifndef USE_EVP
26
27
28
#if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
#define USE_EVP 0
#else
Francis Dupont's avatar
Francis Dupont committed
29
30
#define USE_EVP 1
#endif
31
#endif
32
33
34
35
36


#include <isc/entropy.h>
#include <isc/md5.h>
#include <isc/sha1.h>
37
#include <isc/sha2.h>
38
39
40
41
42
43
44
#include <isc/mem.h>
#include <isc/string.h>
#include <isc/util.h>

#include <dst/result.h>

#include "dst_internal.h"
45
#include "dst_openssl.h"
46
47
48
49
50
#include "dst_parse.h"

#include <openssl/err.h>
#include <openssl/objects.h>
#include <openssl/rsa.h>
51
52
53
#if OPENSSL_VERSION_NUMBER > 0x00908000L
#include <openssl/bn.h>
#endif
54
#ifdef USE_ENGINE
Francis Dupont's avatar
Francis Dupont committed
55
#include <openssl/engine.h>
56
#endif
57

58
59
60
61
62
63
64
/*
 * Limit the size of public exponents.
 */
#ifndef RSA_MAX_PUBEXP_BITS
#define RSA_MAX_PUBEXP_BITS    35
#endif

65
66
67
68
/*
 * We don't use configure for windows so enforce the OpenSSL version
 * here.  Unlike with configure we don't support overriding this test.
 */
Mark Andrews's avatar
Mark Andrews committed
69
#ifdef WIN32
70
#if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
71
       OPENSSL_VERSION_NUMBER < 0x00908000L) || \
Automatic Updater's avatar
Automatic Updater committed
72
      OPENSSL_VERSION_NUMBER >= 0x0090804fL)
73
74
75
76
77
#error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
#endif
#endif


78
	/*
Francis Dupont's avatar
Francis Dupont committed
79
80
	 * XXXMPA  Temporarily disable RSA_BLINDING as it requires
	 * good quality random data that cannot currently be guaranteed.
81
82
83
84
85
	 * XXXMPA  Find which versions of openssl use pseudo random data
	 * and set RSA_FLAG_BLINDING for those.
	 */

#if 0
86
87
#if OPENSSL_VERSION_NUMBER < 0x0090601fL
#define SET_FLAGS(rsa) \
88
89
90
91
	do { \
	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
	(rsa)->flags |= RSA_FLAG_BLINDING; \
	} while (0)
92
#else
93
94
95
96
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags |= RSA_FLAG_BLINDING; \
	} while (0)
97
#endif
98
99
100
101
102
103
104
105
#endif

#if OPENSSL_VERSION_NUMBER < 0x0090601fL
#define SET_FLAGS(rsa) \
	do { \
	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
	(rsa)->flags &= ~RSA_FLAG_BLINDING; \
	} while (0)
106
107
108
109
110
111
#elif defined(RSA_FLAG_NO_BLINDING)
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
		(rsa)->flags |= RSA_FLAG_NO_BLINDING; \
	} while (0)
112
113
114
115
116
117
#else
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
	} while (0)
#endif
118

Francis Dupont's avatar
Francis Dupont committed
119
120
#define DST_RET(a) {ret = a; goto err;}

121
122
123
124
static isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);

static isc_result_t
opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
Mark Andrews's avatar
Mark Andrews committed
125
126
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx;
127
	const EVP_MD *type = NULL;
Mark Andrews's avatar
Mark Andrews committed
128
129
#endif

130
	UNUSED(key);
131
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
132
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
133
134
135
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
136

Francis Dupont's avatar
Francis Dupont committed
137
138
139
140
141
#if USE_EVP
	evp_md_ctx = EVP_MD_CTX_create();
	if (evp_md_ctx == NULL)
		return (ISC_R_NOMEMORY);

142
143
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
Francis Dupont's avatar
Francis Dupont committed
144
		type = EVP_md5();	/* MD5 + RSA */
145
146
147
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
Francis Dupont's avatar
Francis Dupont committed
148
		type = EVP_sha1();	/* SHA1 + RSA */
149
		break;
150
#ifdef HAVE_EVP_SHA256
151
152
153
	case DST_ALG_RSASHA256:
		type = EVP_sha256();	/* SHA256 + RSA */
		break;
154
155
#endif
#ifdef HAVE_EVP_SHA512
156
157
158
	case DST_ALG_RSASHA512:
		type = EVP_sha512();
		break;
159
#endif
160
161
162
	default:
		INSIST(0);
	}
Francis Dupont's avatar
Francis Dupont committed
163
164
165
166
167
168
169

	if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
		EVP_MD_CTX_destroy(evp_md_ctx);
		return (ISC_R_FAILURE);
	}
	dctx->ctxdata.evp_md_ctx = evp_md_ctx;
#else
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx;

			md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
			if (md5ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_md5_init(md5ctx);
			dctx->ctxdata.md5ctx = md5ctx;
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx;

			sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
			if (sha1ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha1_init(sha1ctx);
			dctx->ctxdata.sha1ctx = sha1ctx;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx;

Automatic Updater's avatar
Automatic Updater committed
198
			sha256ctx = isc_mem_get(dctx->mctx,
199
200
201
202
203
204
205
206
207
208
209
						sizeof(isc_sha256_t));
			if (sha256ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha256_init(sha256ctx);
			dctx->ctxdata.sha256ctx = sha256ctx;
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx;

Automatic Updater's avatar
Automatic Updater committed
210
			sha512ctx = isc_mem_get(dctx->mctx,
211
212
213
214
215
216
217
218
219
						sizeof(isc_sha512_t));
			if (sha512ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha512_init(sha512ctx);
			dctx->ctxdata.sha512ctx = sha512ctx;
		}
		break;
	default:
		INSIST(0);
220
	}
Francis Dupont's avatar
Francis Dupont committed
221
#endif
222
223
224
225
226
227

	return (ISC_R_SUCCESS);
}

static void
opensslrsa_destroyctx(dst_context_t *dctx) {
Mark Andrews's avatar
Mark Andrews committed
228
229
230
231
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
#endif

232
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
233
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
234
235
236
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
237

Francis Dupont's avatar
Francis Dupont committed
238
239
240
241
242
243
#if USE_EVP
	if (evp_md_ctx != NULL) {
		EVP_MD_CTX_destroy(evp_md_ctx);
		dctx->ctxdata.evp_md_ctx = NULL;
	}
#else
244
245
246
247
248
249
250
251
252
253
254
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			if (md5ctx != NULL) {
				isc_md5_invalidate(md5ctx);
				isc_mem_put(dctx->mctx, md5ctx,
					    sizeof(isc_md5_t));
				dctx->ctxdata.md5ctx = NULL;
			}
255
		}
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			if (sha1ctx != NULL) {
				isc_sha1_invalidate(sha1ctx);
				isc_mem_put(dctx->mctx, sha1ctx,
					    sizeof(isc_sha1_t));
				dctx->ctxdata.sha1ctx = NULL;
			}
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			if (sha256ctx != NULL) {
				isc_sha256_invalidate(sha256ctx);
				isc_mem_put(dctx->mctx, sha256ctx,
					    sizeof(isc_sha256_t));
				dctx->ctxdata.sha256ctx = NULL;
			}
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			if (sha512ctx != NULL) {
				isc_sha512_invalidate(sha512ctx);
				isc_mem_put(dctx->mctx, sha512ctx,
					    sizeof(isc_sha512_t));
				dctx->ctxdata.sha512ctx = NULL;
			}
292
		}
293
294
295
		break;
	default:
		INSIST(0);
296
	}
Francis Dupont's avatar
Francis Dupont committed
297
#endif
298
299
300
301
}

static isc_result_t
opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
Mark Andrews's avatar
Mark Andrews committed
302
303
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
Automatic Updater's avatar
Automatic Updater committed
304
#endif
Mark Andrews's avatar
Mark Andrews committed
305

306
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
307
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
308
309
310
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
311

Francis Dupont's avatar
Francis Dupont committed
312
313
314
315
316
#if USE_EVP
	if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
		return (ISC_R_FAILURE);
	}
#else
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_update(md5ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_update(sha1ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_update(sha256ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_update(sha512ctx, data->base, data->length);
		}
		break;
	default:
		INSIST(0);
349
	}
Francis Dupont's avatar
Francis Dupont committed
350
#endif
351
352
353
	return (ISC_R_SUCCESS);
}

354
355
#if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
/*
Mark Andrews's avatar
RFC5702    
Mark Andrews committed
356
 * Digest prefixes from RFC 5702.
357
358
359
360
361
362
363
364
365
366
367
368
 */
static unsigned char sha256_prefix[] =
	 { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
	   0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
static unsigned char sha512_prefix[] =
	 { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
	   0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
#define PREFIXLEN sizeof(sha512_prefix)
#else
#define PREFIXLEN 0
#endif

369
370
371
372
static isc_result_t
opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
	dst_key_t *key = dctx->key;
	isc_region_t r;
Francis Dupont's avatar
Francis Dupont committed
373
374
375
376
377
378
	unsigned int siglen = 0;
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
	EVP_PKEY *pkey = key->keydata.pkey;
#else
	RSA *rsa = key->keydata.rsa;
379
	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
380
	unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
381
	int status;
382
383
	int type = 0;
	unsigned int digestlen = 0;
384
385
386
387
	char *message;
	unsigned long err;
	const char* file;
	int line;
388
389
390
391
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	unsigned int prefixlen = 0;
	const unsigned char *prefix = NULL;
#endif
Francis Dupont's avatar
Francis Dupont committed
392
#endif
393

394
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
395
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
396
397
398
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
399

400
401
	isc_buffer_availableregion(sig, &r);

Francis Dupont's avatar
Francis Dupont committed
402
403
404
405
406
407
408
409
#if USE_EVP
	if (r.length < (unsigned int) EVP_PKEY_size(pkey))
		return (ISC_R_NOSPACE);

	if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
		return (ISC_R_FAILURE);
	}
#else
410
411
412
	if (r.length < (unsigned int) RSA_size(rsa))
		return (ISC_R_NOSPACE);

413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_final(md5ctx, digest);
			type = NID_md5;
			digestlen = ISC_MD5_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_final(sha1ctx, digest);
			type = NID_sha1;
			digestlen = ISC_SHA1_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_final(digest, sha256ctx);
			digestlen = ISC_SHA256_DIGESTLENGTH;
439
440
441
442
443
444
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha256_prefix;
			prefixlen = sizeof(sha256_prefix);
#else
			type = NID_sha256;
#endif
445
446
447
448
449
450
451
452
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_final(digest, sha512ctx);
			digestlen = ISC_SHA512_DIGESTLENGTH;
453
454
455
456
457
458
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha512_prefix;
			prefixlen = sizeof(sha512_prefix);
#else
			type = NID_sha512;
#endif
459
460
461
462
		}
		break;
	default:
		INSIST(0);
463
464
	}

465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		INSIST(type != 0);
		status = RSA_sign(type, digest, digestlen, r.base,
				  &siglen, rsa);
		break;

	case DST_ALG_RSASHA256:
	case DST_ALG_RSASHA512:
		INSIST(prefix != NULL);
		INSIST(prefixlen != 0);
		INSIST(prefixlen + digestlen <= sizeof(digest));

		memmove(digest + prefixlen, digest, digestlen);
		memcpy(digest, prefix, prefixlen);
		status = RSA_private_encrypt(digestlen + prefixlen,
					     digest, r.base, rsa,
					     RSA_PKCS1_PADDING);
		if (status < 0)
			status = 0;
		else
			siglen = status;
		break;

	default:
		INSIST(0);
	}
#else
	INSIST(type != 0);
497
	status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
498
#endif
499
500
	if (status == 0) {
		err = ERR_peek_error_line(&file, &line);
Mark Andrews's avatar
Mark Andrews committed
501
		if (err != 0U) {
502
503
			message = ERR_error_string(err, NULL);
		}
504
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
505
	}
Francis Dupont's avatar
Francis Dupont committed
506
#endif
507
508
509
510
511
512
513

	isc_buffer_add(sig, siglen);

	return (ISC_R_SUCCESS);
}

static isc_result_t
514
opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
515
	dst_key_t *key = dctx->key;
Francis Dupont's avatar
Francis Dupont committed
516
517
518
519
	int status = 0;
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
	EVP_PKEY *pkey = key->keydata.pkey;
520
521
	RSA *rsa;
	int bits;
Francis Dupont's avatar
Francis Dupont committed
522
#else
523
524
525
526
	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
	unsigned char digest[ISC_SHA512_DIGESTLENGTH];
	int type = 0;
	unsigned int digestlen = 0;
Francis Dupont's avatar
Francis Dupont committed
527
	RSA *rsa = key->keydata.rsa;
528
529
530
531
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	unsigned int prefixlen = 0;
	const unsigned char *prefix = NULL;
#endif
Francis Dupont's avatar
Francis Dupont committed
532
#endif
533

534
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
535
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
536
537
538
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
539

Francis Dupont's avatar
Francis Dupont committed
540
#if USE_EVP
541
542
543
544
545
546
547
548
	rsa = EVP_PKEY_get1_RSA(pkey);
	if (rsa == NULL)
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	bits = BN_num_bits(rsa->e);
	RSA_free(rsa);
	if (bits > maxbits && maxbits != 0)
		return (DST_R_VERIFYFAILURE);

Francis Dupont's avatar
Francis Dupont committed
549
550
	status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
#else
551
552
553
	if (BN_num_bits(rsa->e) > maxbits && maxbits != 0)
		return (DST_R_VERIFYFAILURE);

554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_final(md5ctx, digest);
			type = NID_md5;
			digestlen = ISC_MD5_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_final(sha1ctx, digest);
			type = NID_sha1;
			digestlen = ISC_SHA1_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_final(digest, sha256ctx);
			digestlen = ISC_SHA256_DIGESTLENGTH;
580
581
582
583
584
585
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha256_prefix;
			prefixlen = sizeof(sha256_prefix);
#else
			type = NID_sha256;
#endif
586
587
588
589
590
591
592
593
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_final(digest, sha512ctx);
			digestlen = ISC_SHA512_DIGESTLENGTH;
594
595
596
597
598
599
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha512_prefix;
			prefixlen = sizeof(sha512_prefix);
#else
			type = NID_sha512;
#endif
600
601
602
603
		}
		break;
	default:
		INSIST(0);
604
605
	}

606
	if (sig->length != (unsigned int) RSA_size(rsa))
607
608
		return (DST_R_VERIFYFAILURE);

609
610
611
612
613
614
615
616
617
618
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		INSIST(type != 0);
		status = RSA_verify(type, digest, digestlen, sig->base,
				    RSA_size(rsa), rsa);
		break;

Automatic Updater's avatar
Automatic Updater committed
619
	case DST_ALG_RSASHA256:
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
	case DST_ALG_RSASHA512:
		{
			/*
			 * 1024 is big enough for all valid RSA bit sizes
			 * for use with DNSSEC.
			 */
			unsigned char original[PREFIXLEN + 1024];

			INSIST(prefix != NULL);
			INSIST(prefixlen != 0U);

			if (RSA_size(rsa) > (int)sizeof(original))
				return (DST_R_VERIFYFAILURE);

			status = RSA_public_decrypt(sig->length, sig->base,
						    original, rsa,
						    RSA_PKCS1_PADDING);
Automatic Updater's avatar
Automatic Updater committed
637
			if (status <= 0)
638
				return (DST_R_VERIFYFAILURE);
Automatic Updater's avatar
Automatic Updater committed
639
			if (status != (int)(prefixlen + digestlen))
640
641
642
643
644
645
646
647
648
649
650
651
652
653
				return (DST_R_VERIFYFAILURE);
			if (memcmp(original, prefix, prefixlen))
				return (DST_R_VERIFYFAILURE);
			if (memcmp(original + prefixlen, digest, digestlen))
				return (DST_R_VERIFYFAILURE);
			status = 1;
		}
		break;

	default:
		INSIST(0);
	}
#else
	INSIST(type != 0);
654
	status = RSA_verify(type, digest, digestlen, sig->base,
655
656
			     RSA_size(rsa), rsa);
#endif
Francis Dupont's avatar
Francis Dupont committed
657
#endif
Francis Dupont's avatar
Francis Dupont committed
658
	if (status != 1)
659
		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
660
661
662
663

	return (ISC_R_SUCCESS);
}

664
665
666
667
668
static isc_result_t
opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
	return (opensslrsa_verify2(dctx, 0, sig));
}

669
670
671
static isc_boolean_t
opensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
	int status;
Francis Dupont's avatar
Francis Dupont committed
672
673
674
675
	RSA *rsa1 = NULL, *rsa2 = NULL;
#if USE_EVP
	EVP_PKEY *pkey1, *pkey2;
#endif
676

Francis Dupont's avatar
Francis Dupont committed
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
#if USE_EVP
	pkey1 = key1->keydata.pkey;
	pkey2 = key2->keydata.pkey;
	/*
	 * The pkey reference will keep these around after
	 * the RSA_free() call.
	 */
	if (pkey1 != NULL) {
		rsa1 = EVP_PKEY_get1_RSA(pkey1);
		RSA_free(rsa1);
	}
	if (pkey2 != NULL) {
		rsa2 = EVP_PKEY_get1_RSA(pkey2);
		RSA_free(rsa2);
	}
#else
693
694
	rsa1 = key1->keydata.rsa;
	rsa2 = key2->keydata.rsa;
Francis Dupont's avatar
Francis Dupont committed
695
#endif
696
697
698
699
700
701
702
703
704
705
706
707

	if (rsa1 == NULL && rsa2 == NULL)
		return (ISC_TRUE);
	else if (rsa1 == NULL || rsa2 == NULL)
		return (ISC_FALSE);

	status = BN_cmp(rsa1->n, rsa2->n) ||
		 BN_cmp(rsa1->e, rsa2->e);

	if (status != 0)
		return (ISC_FALSE);

Francis Dupont's avatar
Francis Dupont committed
708
709
710
711
712
713
714
715
716
717
718
719
720
#if USE_EVP
	if ((rsa1->flags & RSA_FLAG_EXT_PKEY) != 0 ||
	    (rsa2->flags & RSA_FLAG_EXT_PKEY) != 0) {
		if ((rsa1->flags & RSA_FLAG_EXT_PKEY) == 0 ||
		    (rsa2->flags & RSA_FLAG_EXT_PKEY) == 0)
			return (ISC_FALSE);
		/*
		 * Can't compare private parameters, BTW does it make sense?
		 */
		return (ISC_TRUE);
	}
#endif

721
722
723
724
725
726
727
728
729
730
731
732
733
	if (rsa1->d != NULL || rsa2->d != NULL) {
		if (rsa1->d == NULL || rsa2->d == NULL)
			return (ISC_FALSE);
		status = BN_cmp(rsa1->d, rsa2->d) ||
			 BN_cmp(rsa1->p, rsa2->p) ||
			 BN_cmp(rsa1->q, rsa2->q);

		if (status != 0)
			return (ISC_FALSE);
	}
	return (ISC_TRUE);
}

734
735
736
737
#if OPENSSL_VERSION_NUMBER > 0x00908000L
static int
progress_cb(int p, int n, BN_GENCB *cb)
{
738
739
740
741
	union {
		void *dptr;
		void (*fptr)(int);
	} u;
742
743

	UNUSED(n);
744
745
746
747

	u.dptr = cb->arg;
	if (u.fptr != NULL)
		u.fptr(p);
748
749
750
751
	return (1);
}
#endif

752
static isc_result_t
753
opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
754
#if OPENSSL_VERSION_NUMBER > 0x00908000L
755
	BN_GENCB cb;
756
757
758
759
	union {
		void *dptr;
		void (*fptr)(int);
	} u;
760
761
	RSA *rsa = RSA_new();
	BIGNUM *e = BN_new();
Francis Dupont's avatar
Francis Dupont committed
762
763
764
#if USE_EVP
	EVP_PKEY *pkey = EVP_PKEY_new();
#endif
765

766
	if (rsa == NULL || e == NULL)
767
		goto err;
Francis Dupont's avatar
Francis Dupont committed
768
769
770
771
772
773
#if USE_EVP
	if (pkey == NULL)
		goto err;
	if (!EVP_PKEY_set1_RSA(pkey, rsa))
		goto err;
#endif
774

775
776
777
778
779
	if (exp == 0) {
		/* RSA_F4 0x10001 */
		BN_set_bit(e, 0);
		BN_set_bit(e, 16);
	} else {
780
		/* (phased-out) F5 0x100000001 */
781
		BN_set_bit(e, 0);
782
		BN_set_bit(e, 32);
783
784
	}

785
786
787
	if (callback == NULL) {
		BN_GENCB_set_old(&cb, NULL, NULL);
	} else {
788
789
		u.fptr = callback;
		BN_GENCB_set(&cb, &progress_cb, u.dptr);
790
	}
791

792
793
	if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
		BN_free(e);
794
		SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
795
796
#if USE_EVP
		key->keydata.pkey = pkey;
Automatic Updater's avatar
Automatic Updater committed
797

Francis Dupont's avatar
Francis Dupont committed
798
799
		RSA_free(rsa);
#else
800
		key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
801
#endif
802
		return (ISC_R_SUCCESS);
803
	}
804

805
err:
Francis Dupont's avatar
Francis Dupont committed
806
807
808
809
#if USE_EVP
	if (pkey != NULL)
		EVP_PKEY_free(pkey);
#endif
810
	if (e != NULL)
811
		BN_free(e);
812
	if (rsa != NULL)
813
		RSA_free(rsa);
814
	return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
815
#else
816
817
	RSA *rsa;
	unsigned long e;
Francis Dupont's avatar
Francis Dupont committed
818
819
#if USE_EVP
	EVP_PKEY *pkey = EVP_PKEY_new();
Automatic Updater's avatar
Automatic Updater committed
820

821
822
	UNUSED(callback);

Francis Dupont's avatar
Francis Dupont committed
823
824
	if (pkey == NULL)
		return (ISC_R_NOMEMORY);
825
826
#else
	UNUSED(callback);
Francis Dupont's avatar
Francis Dupont committed
827
#endif
828
829

	if (exp == 0)
830
	       e = RSA_F4;
831
	else
832
	       e = 0x40000003;
833
	rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
Francis Dupont's avatar
Francis Dupont committed
834
835
836
837
838
839
	if (rsa == NULL) {
#if USE_EVP
		EVP_PKEY_free(pkey);
#endif
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	}
840
	SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
841
842
843
844
845
846
847
848
849
#if USE_EVP
	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
		EVP_PKEY_free(pkey);
		RSA_free(rsa);
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	}
	key->keydata.pkey = pkey;
	RSA_free(rsa);
#else
850
	key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
851
#endif
852
853

	return (ISC_R_SUCCESS);
854
#endif
855
856
857
858
}

static isc_boolean_t
opensslrsa_isprivate(const dst_key_t *key) {
Francis Dupont's avatar
Francis Dupont committed
859
860
861
862
863
864
865
866
867
868
#if USE_EVP
	RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
	INSIST(rsa != NULL);
	RSA_free(rsa);
	/* key->keydata.pkey still has a reference so rsa is still valid. */
#else
	RSA *rsa = key->keydata.rsa;
#endif
	if (rsa != NULL && (rsa->flags & RSA_FLAG_EXT_PKEY) != 0)
		return (ISC_TRUE);
Brian Wellington's avatar
Brian Wellington committed
869
	return (ISC_TF(rsa != NULL && rsa->d != NULL));
870
871
872
873
}

static void
opensslrsa_destroy(dst_key_t *key) {
Francis Dupont's avatar
Francis Dupont committed
874
875
876
877
878
#if USE_EVP
	EVP_PKEY *pkey = key->keydata.pkey;
	EVP_PKEY_free(pkey);
	key->keydata.pkey = NULL;
#else
879
	RSA *rsa = key->keydata.rsa;
880
	RSA_free(rsa);
881
	key->keydata.rsa = NULL;
Francis Dupont's avatar
Francis Dupont committed
882
#endif
883
884
885
886
887
888
889
890
}


static isc_result_t
opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data) {
	isc_region_t r;
	unsigned int e_bytes;
	unsigned int mod_bytes;
Francis Dupont's avatar
Francis Dupont committed
891
892
893
894
895
	isc_result_t ret;
	RSA *rsa;
#if USE_EVP
	EVP_PKEY *pkey;
#endif
896

Francis Dupont's avatar
Francis Dupont committed
897
898
899
#if USE_EVP
	REQUIRE(key->keydata.pkey != NULL);
#else
900
	REQUIRE(key->keydata.rsa != NULL);
Francis Dupont's avatar
Francis Dupont committed
901
#endif
902

Francis Dupont's avatar
Francis Dupont committed
903
904
905
906
907
908
#if USE_EVP
	pkey = key->keydata.pkey;
	rsa = EVP_PKEY_get1_RSA(pkey);
	if (rsa == NULL)
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
#else
909
	rsa = key->keydata.rsa;
Francis Dupont's avatar
Francis Dupont committed
910
#endif
911
912
913
914
915
916

	isc_buffer_availableregion(data, &r);

	e_bytes = BN_num_bytes(rsa->e);
	mod_bytes = BN_num_bytes(rsa->n);

917
	if (e_bytes < 256) {	/*%< key exponent is <= 2040 bits */
918
		if (r.length < 1)
Francis Dupont's avatar
Francis Dupont committed
919
			DST_RET(ISC_R_NOSPACE);
920
		isc_buffer_putuint8(data, (isc_uint8_t) e_bytes);
921
		isc_region_consume(&r, 1);
922
923
	} else {
		if (r.length < 3)
Francis Dupont's avatar
Francis Dupont committed
924
			DST_RET(ISC_R_NOSPACE);
925
926
		isc_buffer_putuint8(data, 0);
		isc_buffer_putuint16(data, (isc_uint16_t) e_bytes);
927
		isc_region_consume(&r, 3);
928
929
930
	}

	if (r.length < e_bytes + mod_bytes)
931
		DST_RET(ISC_R_NOSPACE);
932
933

	BN_bn2bin(rsa->e, r.base);
934
	isc_region_consume(&r, e_bytes);
935
936
937
938
	BN_bn2bin(rsa->n, r.base);

	isc_buffer_add(data, e_bytes + mod_bytes);

Francis Dupont's avatar
Francis Dupont committed
939
940
941
942
943
944
945
	ret = ISC_R_SUCCESS;
 err:
#if USE_EVP
	if (rsa != NULL)
		RSA_free(rsa);
#endif
	return (ret);
946
947
948
949
950
951
952
}

static isc_result_t
opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
	RSA *rsa;
	isc_region_t r;
	unsigned int e_bytes;
Francis Dupont's avatar
Francis Dupont committed
953
954
955
#if USE_EVP
	EVP_PKEY *pkey;
#endif
956
957
958
959
960
961
962

	isc_buffer_remainingregion(data, &r);
	if (r.length == 0)
		return (ISC_R_SUCCESS);

	rsa = RSA_new();
	if (rsa == NULL)
963
		return (dst__openssl_toresult(ISC_R_NOMEMORY));
964
	SET_FLAGS(rsa);
965

Brian Wellington's avatar
Brian Wellington committed
966
967
	if (r.length < 1) {
		RSA_free(rsa);
968
		return (DST_R_INVALIDPUBLICKEY);
Brian Wellington's avatar
Brian Wellington committed
969
	}
970
971
972
973
	e_bytes = *r.base++;
	r.length--;

	if (e_bytes == 0) {
Brian Wellington's avatar
Brian Wellington committed
974
975
		if (r.length < 2) {
			RSA_free(rsa);
976
			return (DST_R_INVALIDPUBLICKEY);
Brian Wellington's avatar
Brian Wellington committed
977
		}
978
979
980
981
982
		e_bytes = ((*r.base++) << 8);
		e_bytes += *r.base++;
		r.length -= 2;
	}

Brian Wellington's avatar
Brian Wellington committed
983
984
	if (r.length < e_bytes) {
		RSA_free(rsa);
985
		return (DST_R_INVALIDPUBLICKEY);
Brian Wellington's avatar
Brian Wellington committed
986
	}
987
988
989
990
991
992
993
994
995
996
	rsa->e = BN_bin2bn(r.base, e_bytes, NULL);
	r.base += e_bytes;
	r.length -= e_bytes;

	rsa->n = BN_bin2bn(r.base, r.length, NULL);

	key->key_size = BN_num_bits(rsa->n);

	isc_buffer_forward(data, r.length);

Francis Dupont's avatar
Francis Dupont committed
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
#if USE_EVP
	pkey = EVP_PKEY_new();
	if (pkey == NULL) {
		RSA_free(rsa);
		return (ISC_R_NOMEMORY);
	}
	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
		EVP_PKEY_free(pkey);
		RSA_free(rsa);
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	}
	key->keydata.pkey = pkey;
	RSA_free(rsa);
#else
1011
	key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
1012
#endif
1013
1014
1015
1016
1017
1018

	return (ISC_R_SUCCESS);
}

static isc_result_t
opensslrsa_tofile(const dst_key_t *key, const char *directory) {
1019
	int i;
1020
1021
	RSA *rsa;
	dst_private_t priv;
1022
1023
	unsigned char *bufs[8];
	isc_result_t result;
1024

Francis Dupont's avatar
Francis Dupont committed
1025
1026
1027
1028
1029
1030
1031
#if USE_EVP
	if (key->keydata.pkey == NULL)
		return (DST_R_NULLKEY);
	rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
	if (rsa == NULL)
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
#else
1032
	if (key->keydata.rsa == NULL)
1033
		return (DST_R_NULLKEY);
1034
	rsa = key->keydata.rsa;
Francis Dupont's avatar
Francis Dupont committed
1035
#endif
1036

1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
	for (i = 0; i < 8; i++) {
		bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
		if (bufs[i] == NULL) {
			result = ISC_R_NOMEMORY;
			goto fail;
		}
	}

	i = 0;

	priv.elements[i].tag = TAG_RSA_MODULUS;
	priv.elements[i].length = BN_num_bytes(rsa->n);
	BN_bn2bin(rsa->n, bufs[i]);
	priv.elements[i].data = bufs[i];
	i++;

	priv.elements[i].tag = TAG_RSA_PUBLICEXPONENT;
	priv.elements[i].length = BN_num_bytes(rsa->e);
	BN_bn2bin(rsa->e, bufs[i]);
	priv.elements[i].data = bufs[i];
	i++;

Francis Dupont's avatar
Francis Dupont committed
1059
1060
1061
1062
1063
1064
1065
	if (rsa->d != NULL) {
		priv.elements[i].tag = TAG_RSA_PRIVATEEXPONENT;
		priv.elements[i].length = BN_num_bytes(rsa->d);
		BN_bn2bin(rsa->d, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1066

Francis Dupont's avatar
Francis Dupont committed
1067
1068
1069
1070
1071
1072
1073
	if (rsa->p != NULL) {
		priv.elements[i].tag = TAG_RSA_PRIME1;
		priv.elements[i].length = BN_num_bytes(rsa->p);
		BN_bn2bin(rsa->p, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1074

Francis Dupont's avatar
Francis Dupont committed
1075
1076
1077
1078
1079
1080
1081
	if (rsa->q != NULL) {
		priv.elements[i].tag = TAG_RSA_PRIME2;
		priv.elements[i].length = BN_num_bytes(rsa->q);
		BN_bn2bin(rsa->q, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1082

Francis Dupont's avatar
Francis Dupont committed
1083
1084
1085
1086
1087
1088
1089
	if (rsa->dmp1 != NULL) {
		priv.elements[i].tag = TAG_RSA_EXPONENT1;
		priv.elements[i].length = BN_num_bytes(rsa->dmp1);
		BN_bn2bin(rsa->dmp1, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1090

Francis Dupont's avatar
Francis Dupont committed
1091
1092
1093
1094
1095
1096
1097
	if (rsa->dmq1 != NULL) {
		priv.elements[i].tag = TAG_RSA_EXPONENT2;
		priv.elements[i].length = BN_num_bytes(rsa->dmq1);
		BN_bn2bin(rsa->dmq1, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1098

Francis Dupont's avatar
Francis Dupont committed
1099
1100
1101
1102
1103
1104
1105
	if (rsa->iqmp != NULL) {
		priv.elements[i].tag = TAG_RSA_COEFFICIENT;
		priv.elements[i].length = BN_num_bytes(rsa->iqmp);
		BN_bn2bin(rsa->iqmp, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
Automatic Updater's avatar
Automatic Updater committed
1106

Francis Dupont's avatar
Francis Dupont committed
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
	if (key->engine != NULL) {
		priv.elements[i].tag = TAG_RSA_ENGINE;
		priv.elements[i].length = strlen(key->engine) + 1;
		priv.elements[i].data = (unsigned char *)key->engine;
		i++;
	}

	if (key->label != NULL) {
		priv.elements[i].tag = TAG_RSA_LABEL;
		priv.elements[i].length = strlen(key->label) + 1;
		priv.elements[i].data = (unsigned char *)key->label;
		i++;
	}
1120

1121

1122
	priv.nelements = i;
1123
	result = dst__privstruct_writefile(key, &priv, directory);
1124
 fail:
Francis Dupont's avatar
Francis Dupont committed
1125
1126
1127
#if USE_EVP
	RSA_free(rsa);
#endif
1128
1129
1130
1131
1132
	for (i = 0; i < 8; i++) {
		if (bufs[i] == NULL)
			break;
		isc_mem_put(key->mctx, bufs[i], BN_num_bytes(rsa->n));
	}
1133
	return (result);
1134
1135
1136
}

static isc_result_t
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
rsa_check(RSA *rsa, RSA *pub)
{
	/* Public parameters should be the same but if they are not set
	 * copy them from the public key. */
	if (pub != NULL) {
		if (rsa->n != NULL) {
			if (BN_cmp(rsa->n, pub->n) != 0)
				return (DST_R_INVALIDPRIVATEKEY);
		} else {
			rsa->n = pub->n;
			pub->n = NULL;
		}
		if (rsa->e != NULL) {
			if (BN_cmp(rsa->e, pub->e) != 0)
				return (DST_R_INVALIDPRIVATEKEY);
		} else {
			rsa->e = pub->e;
			pub->e = NULL;
		}
	}
	if (rsa->n == NULL || rsa->e == NULL)
		return (DST_R_INVALIDPRIVATEKEY);
	return (ISC_R_SUCCESS);
}

static isc_result_t
opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
1164
1165
1166
	dst_private_t priv;
	isc_result_t ret;
	int i;
1167
	RSA *rsa = NULL, *pubrsa = NULL;
1168
#ifdef USE_ENGINE
Francis Dupont's avatar
Francis Dupont committed
1169
	ENGINE *e = NULL;
1170
#endif
1171
	isc_mem_t *mctx = key->mctx;
Francis Dupont's avatar
Francis Dupont committed
1172
	const char *engine = NULL, *label = NULL;
1173
#if defined(USE_ENGINE) || USE_EVP
Francis Dupont's avatar
Francis Dupont committed
1174
	EVP_PKEY *pkey = NULL;
1175
#endif
1176

1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
#if USE_EVP
	if (pub != NULL && pub->keydata.pkey != NULL)
		pubrsa = EVP_PKEY_get1_RSA(pub->keydata.pkey);
#else
	if (pub != NULL && pub->keydata.rsa != NULL) {
		pubrsa = pub->keydata.rsa;
		pub->keydata.rsa = NULL;
	}
#endif

1187
	/* read private key file */
1188
	ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
1189
1190
1191
	if (ret != ISC_R_SUCCESS)
		return (ret);

Francis Dupont's avatar
Francis Dupont committed
1192
1193
1194
	for (i = 0; i < priv.nelements; i++) {
		switch (priv.elements[i].tag) {
		case TAG_RSA_ENGINE:
Francis Dupont's avatar
Francis Dupont committed
1195
			engine = (char *)priv.elements[i].data;
Francis Dupont's avatar
Francis Dupont committed
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
			break;
		case TAG_RSA_LABEL:
			label = (char *)priv.elements[i].data;
			break;
		default:
			break;
		}
	}
	/*
	 * Is this key is stored in a HSM?
	 * See if we can fetch it.
	 */
Francis Dupont's avatar
Francis Dupont committed
1208
	if (label != NULL) {
1209
#ifdef USE_ENGINE
Francis Dupont's avatar
Francis Dupont committed
1210
1211
1212
		if (engine == NULL)
			DST_RET(DST_R_NOENGINE);
		e = dst__openssl_getengine(engine);
Francis Dupont's avatar
Francis Dupont committed
1213
1214
1215
1216
		if (e == NULL)
			DST_RET(DST_R_NOENGINE);
		pkey = ENGINE_load_private_key(e, label, NULL, NULL);
		if (pkey == NULL) {
1217
1218
			/* ERR_print_errors_fp(stderr); */
			DST_RET(ISC_R_NOTFOUND);
Francis Dupont's avatar
Francis Dupont committed
1219
		}
Francis Dupont's avatar
Francis Dupont committed
1220
		key->engine = isc_mem_strdup(key->mctx, engine);
Francis Dupont's avatar
Francis Dupont committed
1221
1222
1223
1224
1225
		if (key->engine == NULL)
			DST_RET(ISC_R_NOMEMORY);
		key->label = isc_mem_strdup(key->mctx, label);
		if (key->label == NULL)
			DST_RET(ISC_R_NOMEMORY);
1226
1227
1228
1229
1230
		rsa = EVP_PKEY_get1_RSA(pkey);
		if (rsa == NULL)
			DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
		if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
			DST_RET(DST_R_INVALIDPRIVATEKEY);
1231
1232
		if (BN_num_bits(rsa->e) > RSA_MAX_PUBEXP_BITS)
			DST_RET(ISC_R_RANGE);
Francis Dupont's avatar
Francis Dupont committed
1233
1234
		if (pubrsa != NULL)
			RSA_free(pubrsa);
Francis Dupont's avatar
Francis Dupont committed
1235
1236
1237
		key->key_size = EVP_PKEY_bits(pkey);
#if USE_EVP
		key->keydata.pkey = pkey;
Francis Dupont's avatar
Francis Dupont committed
1238
		RSA_free(rsa);
Francis Dupont's avatar
Francis Dupont committed
1239
#else
1240
		key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
1241
1242
1243
		EVP_PKEY_free(pkey);
#endif
		dst__privstruct_free(&priv, mctx);
1244
		memset(&priv, 0, sizeof(priv));
Francis Dupont's avatar
Francis Dupont committed
1245
		return (ISC_R_SUCCESS);
1246
1247
1248
#else
		DST_RET(DST_R_NOENGINE);
#endif
Automatic Updater's avatar
Automatic Updater committed
1249
	}
Francis Dupont's avatar
Francis Dupont committed
1250

1251
1252
1253
	rsa = RSA_new();
	if (rsa == NULL)
		DST_RET(ISC_R_NOMEMORY);
1254
	SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
1255
1256
1257
1258
1259

#if USE_EVP
	pkey = EVP_PKEY_new();
	if (pkey == NULL)
		DST_RET(ISC_R_NOMEMORY);
Francis Dupont's avatar
Francis Dupont committed
1260
	if (!EVP_PKEY_set1_RSA(pkey, rsa))
Francis Dupont's avatar
Francis Dupont committed
1261
1262
1263
		DST_RET(ISC_R_FAILURE);
	key->keydata.pkey = pkey;
#else
1264
	key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
1265
#endif
1266
1267
1268

	for (i = 0; i < priv.nelements; i++) {
		BIGNUM *bn;
Francis Dupont's avatar
Francis Dupont committed
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
		switch (priv.elements[i].tag) {
		case TAG_RSA_ENGINE:
			continue;
		case TAG_RSA_LABEL:
			continue;
		case TAG_RSA_PIN:
			continue;
		default:
			bn = BN_bin2bn(priv.elements[i].data,
				       priv.elements[i].length, NULL);
			if (bn == NULL)
				DST_RET(ISC_R_NOMEMORY);
Automatic Updater's avatar
Automatic Updater committed
1281
		}
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307

		switch (priv.elements[i].tag) {
			case TAG_RSA_MODULUS:
				rsa->n = bn;
				break;
			case TAG_RSA_PUBLICEXPONENT:
				rsa->e = bn;
				break;
			case TAG_RSA_PRIVATEEXPONENT:
				rsa->d = bn;
				break;
			case TAG_RSA_PRIME1:
				rsa->p = bn;
				break;
			case TAG_RSA_PRIME2:
				rsa->q = bn;
				break;
			case TAG_RSA_EXPONENT1:
				rsa->dmp1 = bn;
				break;
			case TAG_RSA_EXPONENT2:
				rsa->dmq1 = bn;
				break;
			case TAG_RSA_COEFFICIENT:
				rsa->iqmp = bn;
				break;