opensslrsa_link.c 31.1 KB
Newer Older
1
/*
Automatic Updater's avatar
Automatic Updater committed
2
 * Copyright (C) 2004-2009, 2011  Internet Systems Consortium, Inc. ("ISC")
Mark Andrews's avatar
Mark Andrews committed
3
 * Copyright (C) 2000-2003  Internet Software Consortium.
4
 *
Automatic Updater's avatar
Automatic Updater committed
5
 * Permission to use, copy, modify, and/or distribute this software for any
6
7
8
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
Mark Andrews's avatar
Mark Andrews committed
9
10
11
12
13
14
15
 * THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
 * REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
 * AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
 * INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
 * LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
 * OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
 * PERFORMANCE OF THIS SOFTWARE.
16
17
18
19
 */

/*
 * Principal Author: Brian Wellington
Mark Andrews's avatar
Mark Andrews committed
20
 * $Id$
21
 */
22
#ifdef OPENSSL
23
24
#include <config.h>

Francis Dupont's avatar
Francis Dupont committed
25
#ifndef USE_EVP
26
27
28
#if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
#define USE_EVP 0
#else
Francis Dupont's avatar
Francis Dupont committed
29
30
#define USE_EVP 1
#endif
31
#endif
32
33
34
35
36


#include <isc/entropy.h>
#include <isc/md5.h>
#include <isc/sha1.h>
37
#include <isc/sha2.h>
38
39
40
41
42
43
44
#include <isc/mem.h>
#include <isc/string.h>
#include <isc/util.h>

#include <dst/result.h>

#include "dst_internal.h"
45
#include "dst_openssl.h"
46
47
48
49
50
#include "dst_parse.h"

#include <openssl/err.h>
#include <openssl/objects.h>
#include <openssl/rsa.h>
51
52
53
#if OPENSSL_VERSION_NUMBER > 0x00908000L
#include <openssl/bn.h>
#endif
54
#ifdef USE_ENGINE
Francis Dupont's avatar
Francis Dupont committed
55
#include <openssl/engine.h>
56
#endif
57

58
59
60
61
/*
 * We don't use configure for windows so enforce the OpenSSL version
 * here.  Unlike with configure we don't support overriding this test.
 */
Mark Andrews's avatar
Mark Andrews committed
62
#ifdef WIN32
63
#if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
64
       OPENSSL_VERSION_NUMBER < 0x00908000L) || \
Automatic Updater's avatar
Automatic Updater committed
65
      OPENSSL_VERSION_NUMBER >= 0x0090804fL)
66
67
68
69
70
#error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
#endif
#endif


71
	/*
Francis Dupont's avatar
Francis Dupont committed
72
73
	 * XXXMPA  Temporarily disable RSA_BLINDING as it requires
	 * good quality random data that cannot currently be guaranteed.
74
75
76
77
78
	 * XXXMPA  Find which versions of openssl use pseudo random data
	 * and set RSA_FLAG_BLINDING for those.
	 */

#if 0
79
80
#if OPENSSL_VERSION_NUMBER < 0x0090601fL
#define SET_FLAGS(rsa) \
81
82
83
84
	do { \
	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
	(rsa)->flags |= RSA_FLAG_BLINDING; \
	} while (0)
85
#else
86
87
88
89
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags |= RSA_FLAG_BLINDING; \
	} while (0)
90
#endif
91
92
93
94
95
96
97
98
#endif

#if OPENSSL_VERSION_NUMBER < 0x0090601fL
#define SET_FLAGS(rsa) \
	do { \
	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
	(rsa)->flags &= ~RSA_FLAG_BLINDING; \
	} while (0)
99
100
101
102
103
104
#elif defined(RSA_FLAG_NO_BLINDING)
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
		(rsa)->flags |= RSA_FLAG_NO_BLINDING; \
	} while (0)
105
106
107
108
109
110
#else
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
	} while (0)
#endif
111

Francis Dupont's avatar
Francis Dupont committed
112
113
#define DST_RET(a) {ret = a; goto err;}

114
115
116
117
static isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);

static isc_result_t
opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
Mark Andrews's avatar
Mark Andrews committed
118
119
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx;
120
	const EVP_MD *type = NULL;
Mark Andrews's avatar
Mark Andrews committed
121
122
#endif

123
	UNUSED(key);
124
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
125
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
126
127
128
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
129

Francis Dupont's avatar
Francis Dupont committed
130
131
132
133
134
#if USE_EVP
	evp_md_ctx = EVP_MD_CTX_create();
	if (evp_md_ctx == NULL)
		return (ISC_R_NOMEMORY);

135
136
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
Francis Dupont's avatar
Francis Dupont committed
137
		type = EVP_md5();	/* MD5 + RSA */
138
139
140
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
Francis Dupont's avatar
Francis Dupont committed
141
		type = EVP_sha1();	/* SHA1 + RSA */
142
		break;
143
#ifdef HAVE_EVP_SHA256
144
145
146
	case DST_ALG_RSASHA256:
		type = EVP_sha256();	/* SHA256 + RSA */
		break;
147
148
#endif
#ifdef HAVE_EVP_SHA512
149
150
151
	case DST_ALG_RSASHA512:
		type = EVP_sha512();
		break;
152
#endif
153
154
155
	default:
		INSIST(0);
	}
Francis Dupont's avatar
Francis Dupont committed
156
157
158
159
160
161
162

	if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
		EVP_MD_CTX_destroy(evp_md_ctx);
		return (ISC_R_FAILURE);
	}
	dctx->ctxdata.evp_md_ctx = evp_md_ctx;
#else
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx;

			md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
			if (md5ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_md5_init(md5ctx);
			dctx->ctxdata.md5ctx = md5ctx;
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx;

			sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
			if (sha1ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha1_init(sha1ctx);
			dctx->ctxdata.sha1ctx = sha1ctx;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx;

Automatic Updater's avatar
Automatic Updater committed
191
			sha256ctx = isc_mem_get(dctx->mctx,
192
193
194
195
196
197
198
199
200
201
202
						sizeof(isc_sha256_t));
			if (sha256ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha256_init(sha256ctx);
			dctx->ctxdata.sha256ctx = sha256ctx;
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx;

Automatic Updater's avatar
Automatic Updater committed
203
			sha512ctx = isc_mem_get(dctx->mctx,
204
205
206
207
208
209
210
211
212
						sizeof(isc_sha512_t));
			if (sha512ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha512_init(sha512ctx);
			dctx->ctxdata.sha512ctx = sha512ctx;
		}
		break;
	default:
		INSIST(0);
213
	}
Francis Dupont's avatar
Francis Dupont committed
214
#endif
215
216
217
218
219
220

	return (ISC_R_SUCCESS);
}

static void
opensslrsa_destroyctx(dst_context_t *dctx) {
Mark Andrews's avatar
Mark Andrews committed
221
222
223
224
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
#endif

225
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
226
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
227
228
229
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
230

Francis Dupont's avatar
Francis Dupont committed
231
232
233
234
235
236
#if USE_EVP
	if (evp_md_ctx != NULL) {
		EVP_MD_CTX_destroy(evp_md_ctx);
		dctx->ctxdata.evp_md_ctx = NULL;
	}
#else
237
238
239
240
241
242
243
244
245
246
247
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			if (md5ctx != NULL) {
				isc_md5_invalidate(md5ctx);
				isc_mem_put(dctx->mctx, md5ctx,
					    sizeof(isc_md5_t));
				dctx->ctxdata.md5ctx = NULL;
			}
248
		}
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			if (sha1ctx != NULL) {
				isc_sha1_invalidate(sha1ctx);
				isc_mem_put(dctx->mctx, sha1ctx,
					    sizeof(isc_sha1_t));
				dctx->ctxdata.sha1ctx = NULL;
			}
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			if (sha256ctx != NULL) {
				isc_sha256_invalidate(sha256ctx);
				isc_mem_put(dctx->mctx, sha256ctx,
					    sizeof(isc_sha256_t));
				dctx->ctxdata.sha256ctx = NULL;
			}
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			if (sha512ctx != NULL) {
				isc_sha512_invalidate(sha512ctx);
				isc_mem_put(dctx->mctx, sha512ctx,
					    sizeof(isc_sha512_t));
				dctx->ctxdata.sha512ctx = NULL;
			}
285
		}
286
287
288
		break;
	default:
		INSIST(0);
289
	}
Francis Dupont's avatar
Francis Dupont committed
290
#endif
291
292
293
294
}

static isc_result_t
opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
Mark Andrews's avatar
Mark Andrews committed
295
296
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
Automatic Updater's avatar
Automatic Updater committed
297
#endif
Mark Andrews's avatar
Mark Andrews committed
298

299
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
300
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
301
302
303
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
304

Francis Dupont's avatar
Francis Dupont committed
305
306
307
308
309
#if USE_EVP
	if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
		return (ISC_R_FAILURE);
	}
#else
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_update(md5ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_update(sha1ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_update(sha256ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_update(sha512ctx, data->base, data->length);
		}
		break;
	default:
		INSIST(0);
342
	}
Francis Dupont's avatar
Francis Dupont committed
343
#endif
344
345
346
	return (ISC_R_SUCCESS);
}

347
348
#if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
/*
Mark Andrews's avatar
RFC5702    
Mark Andrews committed
349
 * Digest prefixes from RFC 5702.
350
351
352
353
354
355
356
357
358
359
360
361
 */
static unsigned char sha256_prefix[] =
	 { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
	   0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
static unsigned char sha512_prefix[] =
	 { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
	   0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
#define PREFIXLEN sizeof(sha512_prefix)
#else
#define PREFIXLEN 0
#endif

362
363
364
365
static isc_result_t
opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
	dst_key_t *key = dctx->key;
	isc_region_t r;
Francis Dupont's avatar
Francis Dupont committed
366
367
368
369
370
371
	unsigned int siglen = 0;
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
	EVP_PKEY *pkey = key->keydata.pkey;
#else
	RSA *rsa = key->keydata.rsa;
372
	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
373
	unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
374
	int status;
375
376
	int type = 0;
	unsigned int digestlen = 0;
377
378
379
380
	char *message;
	unsigned long err;
	const char* file;
	int line;
381
382
383
384
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	unsigned int prefixlen = 0;
	const unsigned char *prefix = NULL;
#endif
Francis Dupont's avatar
Francis Dupont committed
385
#endif
386

387
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
388
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
389
390
391
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
392

393
394
	isc_buffer_availableregion(sig, &r);

Francis Dupont's avatar
Francis Dupont committed
395
396
397
398
399
400
401
402
#if USE_EVP
	if (r.length < (unsigned int) EVP_PKEY_size(pkey))
		return (ISC_R_NOSPACE);

	if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
		return (ISC_R_FAILURE);
	}
#else
403
404
405
	if (r.length < (unsigned int) RSA_size(rsa))
		return (ISC_R_NOSPACE);

406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_final(md5ctx, digest);
			type = NID_md5;
			digestlen = ISC_MD5_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_final(sha1ctx, digest);
			type = NID_sha1;
			digestlen = ISC_SHA1_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_final(digest, sha256ctx);
			digestlen = ISC_SHA256_DIGESTLENGTH;
432
433
434
435
436
437
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha256_prefix;
			prefixlen = sizeof(sha256_prefix);
#else
			type = NID_sha256;
#endif
438
439
440
441
442
443
444
445
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_final(digest, sha512ctx);
			digestlen = ISC_SHA512_DIGESTLENGTH;
446
447
448
449
450
451
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha512_prefix;
			prefixlen = sizeof(sha512_prefix);
#else
			type = NID_sha512;
#endif
452
453
454
455
		}
		break;
	default:
		INSIST(0);
456
457
	}

458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		INSIST(type != 0);
		status = RSA_sign(type, digest, digestlen, r.base,
				  &siglen, rsa);
		break;

	case DST_ALG_RSASHA256:
	case DST_ALG_RSASHA512:
		INSIST(prefix != NULL);
		INSIST(prefixlen != 0);
		INSIST(prefixlen + digestlen <= sizeof(digest));

		memmove(digest + prefixlen, digest, digestlen);
		memcpy(digest, prefix, prefixlen);
		status = RSA_private_encrypt(digestlen + prefixlen,
					     digest, r.base, rsa,
					     RSA_PKCS1_PADDING);
		if (status < 0)
			status = 0;
		else
			siglen = status;
		break;

	default:
		INSIST(0);
	}
#else
	INSIST(type != 0);
490
	status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
491
#endif
492
493
	if (status == 0) {
		err = ERR_peek_error_line(&file, &line);
Mark Andrews's avatar
Mark Andrews committed
494
		if (err != 0U) {
495
496
			message = ERR_error_string(err, NULL);
		}
497
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
498
	}
Francis Dupont's avatar
Francis Dupont committed
499
#endif
500
501
502
503
504
505
506
507
508

	isc_buffer_add(sig, siglen);

	return (ISC_R_SUCCESS);
}

static isc_result_t
opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
	dst_key_t *key = dctx->key;
Francis Dupont's avatar
Francis Dupont committed
509
510
511
512
513
	int status = 0;
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
	EVP_PKEY *pkey = key->keydata.pkey;
#else
514
515
516
517
	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
	unsigned char digest[ISC_SHA512_DIGESTLENGTH];
	int type = 0;
	unsigned int digestlen = 0;
Francis Dupont's avatar
Francis Dupont committed
518
	RSA *rsa = key->keydata.rsa;
519
520
521
522
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	unsigned int prefixlen = 0;
	const unsigned char *prefix = NULL;
#endif
Francis Dupont's avatar
Francis Dupont committed
523
#endif
524

525
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
526
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
527
528
529
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
530

Francis Dupont's avatar
Francis Dupont committed
531
532
533
#if USE_EVP
	status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
#else
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_final(md5ctx, digest);
			type = NID_md5;
			digestlen = ISC_MD5_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_final(sha1ctx, digest);
			type = NID_sha1;
			digestlen = ISC_SHA1_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_final(digest, sha256ctx);
			digestlen = ISC_SHA256_DIGESTLENGTH;
560
561
562
563
564
565
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha256_prefix;
			prefixlen = sizeof(sha256_prefix);
#else
			type = NID_sha256;
#endif
566
567
568
569
570
571
572
573
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_final(digest, sha512ctx);
			digestlen = ISC_SHA512_DIGESTLENGTH;
574
575
576
577
578
579
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha512_prefix;
			prefixlen = sizeof(sha512_prefix);
#else
			type = NID_sha512;
#endif
580
581
582
583
		}
		break;
	default:
		INSIST(0);
584
585
	}

586
	if (sig->length != (unsigned int) RSA_size(rsa))
587
588
		return (DST_R_VERIFYFAILURE);

589
590
591
592
593
594
595
596
597
598
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		INSIST(type != 0);
		status = RSA_verify(type, digest, digestlen, sig->base,
				    RSA_size(rsa), rsa);
		break;

Automatic Updater's avatar
Automatic Updater committed
599
	case DST_ALG_RSASHA256:
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
	case DST_ALG_RSASHA512:
		{
			/*
			 * 1024 is big enough for all valid RSA bit sizes
			 * for use with DNSSEC.
			 */
			unsigned char original[PREFIXLEN + 1024];

			INSIST(prefix != NULL);
			INSIST(prefixlen != 0U);

			if (RSA_size(rsa) > (int)sizeof(original))
				return (DST_R_VERIFYFAILURE);

			status = RSA_public_decrypt(sig->length, sig->base,
						    original, rsa,
						    RSA_PKCS1_PADDING);
Automatic Updater's avatar
Automatic Updater committed
617
			if (status <= 0)
618
				return (DST_R_VERIFYFAILURE);
Automatic Updater's avatar
Automatic Updater committed
619
			if (status != (int)(prefixlen + digestlen))
620
621
622
623
624
625
626
627
628
629
630
631
632
633
				return (DST_R_VERIFYFAILURE);
			if (memcmp(original, prefix, prefixlen))
				return (DST_R_VERIFYFAILURE);
			if (memcmp(original + prefixlen, digest, digestlen))
				return (DST_R_VERIFYFAILURE);
			status = 1;
		}
		break;

	default:
		INSIST(0);
	}
#else
	INSIST(type != 0);
634
	status = RSA_verify(type, digest, digestlen, sig->base,
635
636
			     RSA_size(rsa), rsa);
#endif
Francis Dupont's avatar
Francis Dupont committed
637
#endif
Francis Dupont's avatar
Francis Dupont committed
638
	if (status != 1)
639
		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
640
641
642
643
644
645
646

	return (ISC_R_SUCCESS);
}

static isc_boolean_t
opensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
	int status;
Francis Dupont's avatar
Francis Dupont committed
647
648
649
650
	RSA *rsa1 = NULL, *rsa2 = NULL;
#if USE_EVP
	EVP_PKEY *pkey1, *pkey2;
#endif
651

Francis Dupont's avatar
Francis Dupont committed
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
#if USE_EVP
	pkey1 = key1->keydata.pkey;
	pkey2 = key2->keydata.pkey;
	/*
	 * The pkey reference will keep these around after
	 * the RSA_free() call.
	 */
	if (pkey1 != NULL) {
		rsa1 = EVP_PKEY_get1_RSA(pkey1);
		RSA_free(rsa1);
	}
	if (pkey2 != NULL) {
		rsa2 = EVP_PKEY_get1_RSA(pkey2);
		RSA_free(rsa2);
	}
#else
668
669
	rsa1 = key1->keydata.rsa;
	rsa2 = key2->keydata.rsa;
Francis Dupont's avatar
Francis Dupont committed
670
#endif
671
672
673
674
675
676
677
678
679
680
681
682

	if (rsa1 == NULL && rsa2 == NULL)
		return (ISC_TRUE);
	else if (rsa1 == NULL || rsa2 == NULL)
		return (ISC_FALSE);

	status = BN_cmp(rsa1->n, rsa2->n) ||
		 BN_cmp(rsa1->e, rsa2->e);

	if (status != 0)
		return (ISC_FALSE);

Francis Dupont's avatar
Francis Dupont committed
683
684
685
686
687
688
689
690
691
692
693
694
695
#if USE_EVP
	if ((rsa1->flags & RSA_FLAG_EXT_PKEY) != 0 ||
	    (rsa2->flags & RSA_FLAG_EXT_PKEY) != 0) {
		if ((rsa1->flags & RSA_FLAG_EXT_PKEY) == 0 ||
		    (rsa2->flags & RSA_FLAG_EXT_PKEY) == 0)
			return (ISC_FALSE);
		/*
		 * Can't compare private parameters, BTW does it make sense?
		 */
		return (ISC_TRUE);
	}
#endif

696
697
698
699
700
701
702
703
704
705
706
707
708
	if (rsa1->d != NULL || rsa2->d != NULL) {
		if (rsa1->d == NULL || rsa2->d == NULL)
			return (ISC_FALSE);
		status = BN_cmp(rsa1->d, rsa2->d) ||
			 BN_cmp(rsa1->p, rsa2->p) ||
			 BN_cmp(rsa1->q, rsa2->q);

		if (status != 0)
			return (ISC_FALSE);
	}
	return (ISC_TRUE);
}

709
710
711
712
#if OPENSSL_VERSION_NUMBER > 0x00908000L
static int
progress_cb(int p, int n, BN_GENCB *cb)
{
713
714
715
716
	union {
		void *dptr;
		void (*fptr)(int);
	} u;
717
718

	UNUSED(n);
719
720
721
722

	u.dptr = cb->arg;
	if (u.fptr != NULL)
		u.fptr(p);
723
724
725
726
	return (1);
}
#endif

727
static isc_result_t
728
opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
729
#if OPENSSL_VERSION_NUMBER > 0x00908000L
730
	BN_GENCB cb;
731
732
733
734
	union {
		void *dptr;
		void (*fptr)(int);
	} u;
735
736
	RSA *rsa = RSA_new();
	BIGNUM *e = BN_new();
Francis Dupont's avatar
Francis Dupont committed
737
738
739
#if USE_EVP
	EVP_PKEY *pkey = EVP_PKEY_new();
#endif
740

741
	if (rsa == NULL || e == NULL)
742
		goto err;
Francis Dupont's avatar
Francis Dupont committed
743
744
745
746
747
748
#if USE_EVP
	if (pkey == NULL)
		goto err;
	if (!EVP_PKEY_set1_RSA(pkey, rsa))
		goto err;
#endif
749

750
751
752
753
754
	if (exp == 0) {
		/* RSA_F4 0x10001 */
		BN_set_bit(e, 0);
		BN_set_bit(e, 16);
	} else {
755
		/* F5 0x100000001 */
756
		BN_set_bit(e, 0);
757
		BN_set_bit(e, 32);
758
759
	}

760
761
762
	if (callback == NULL) {
		BN_GENCB_set_old(&cb, NULL, NULL);
	} else {
763
764
		u.fptr = callback;
		BN_GENCB_set(&cb, &progress_cb, u.dptr);
765
	}
766

767
768
	if (RSA_generate_key_ex(rsa, key->key_size, e, &cb)) {
		BN_free(e);
769
		SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
770
771
#if USE_EVP
		key->keydata.pkey = pkey;
Automatic Updater's avatar
Automatic Updater committed
772

Francis Dupont's avatar
Francis Dupont committed
773
774
		RSA_free(rsa);
#else
775
		key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
776
#endif
777
		return (ISC_R_SUCCESS);
778
	}
779

780
err:
Francis Dupont's avatar
Francis Dupont committed
781
782
783
784
#if USE_EVP
	if (pkey != NULL)
		EVP_PKEY_free(pkey);
#endif
785
	if (e != NULL)
786
		BN_free(e);
787
	if (rsa != NULL)
788
		RSA_free(rsa);
789
	return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
790
#else
791
792
	RSA *rsa;
	unsigned long e;
Francis Dupont's avatar
Francis Dupont committed
793
794
#if USE_EVP
	EVP_PKEY *pkey = EVP_PKEY_new();
Automatic Updater's avatar
Automatic Updater committed
795

796
797
	UNUSED(callback);

Francis Dupont's avatar
Francis Dupont committed
798
799
	if (pkey == NULL)
		return (ISC_R_NOMEMORY);
800
801
#else
	UNUSED(callback);
Francis Dupont's avatar
Francis Dupont committed
802
#endif
803
804

	if (exp == 0)
805
	       e = RSA_F4;
806
	else
807
	       e = 0x40000003;
808
	rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
Francis Dupont's avatar
Francis Dupont committed
809
810
811
812
813
814
	if (rsa == NULL) {
#if USE_EVP
		EVP_PKEY_free(pkey);
#endif
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	}
815
	SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
816
817
818
819
820
821
822
823
824
#if USE_EVP
	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
		EVP_PKEY_free(pkey);
		RSA_free(rsa);
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	}
	key->keydata.pkey = pkey;
	RSA_free(rsa);
#else
825
	key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
826
#endif
827
828

	return (ISC_R_SUCCESS);
829
#endif
830
831
832
833
}

static isc_boolean_t
opensslrsa_isprivate(const dst_key_t *key) {
Francis Dupont's avatar
Francis Dupont committed
834
835
836
837
838
839
840
841
842
843
#if USE_EVP
	RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
	INSIST(rsa != NULL);
	RSA_free(rsa);
	/* key->keydata.pkey still has a reference so rsa is still valid. */
#else
	RSA *rsa = key->keydata.rsa;
#endif
	if (rsa != NULL && (rsa->flags & RSA_FLAG_EXT_PKEY) != 0)
		return (ISC_TRUE);
Brian Wellington's avatar
Brian Wellington committed
844
	return (ISC_TF(rsa != NULL && rsa->d != NULL));
845
846
847
848
}

static void
opensslrsa_destroy(dst_key_t *key) {
Francis Dupont's avatar
Francis Dupont committed
849
850
851
852
853
#if USE_EVP
	EVP_PKEY *pkey = key->keydata.pkey;
	EVP_PKEY_free(pkey);
	key->keydata.pkey = NULL;
#else
854
	RSA *rsa = key->keydata.rsa;
855
	RSA_free(rsa);
856
	key->keydata.rsa = NULL;
Francis Dupont's avatar
Francis Dupont committed
857
#endif
858
859
860
861
862
863
864
865
}


static isc_result_t
opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data) {
	isc_region_t r;
	unsigned int e_bytes;
	unsigned int mod_bytes;
Francis Dupont's avatar
Francis Dupont committed
866
867
868
869
870
	isc_result_t ret;
	RSA *rsa;
#if USE_EVP
	EVP_PKEY *pkey;
#endif
871

Francis Dupont's avatar
Francis Dupont committed
872
873
874
#if USE_EVP
	REQUIRE(key->keydata.pkey != NULL);
#else
875
	REQUIRE(key->keydata.rsa != NULL);
Francis Dupont's avatar
Francis Dupont committed
876
#endif
877

Francis Dupont's avatar
Francis Dupont committed
878
879
880
881
882
883
#if USE_EVP
	pkey = key->keydata.pkey;
	rsa = EVP_PKEY_get1_RSA(pkey);
	if (rsa == NULL)
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
#else
884
	rsa = key->keydata.rsa;
Francis Dupont's avatar
Francis Dupont committed
885
#endif
886
887
888
889
890
891

	isc_buffer_availableregion(data, &r);

	e_bytes = BN_num_bytes(rsa->e);
	mod_bytes = BN_num_bytes(rsa->n);

892
	if (e_bytes < 256) {	/*%< key exponent is <= 2040 bits */
893
		if (r.length < 1)
Francis Dupont's avatar
Francis Dupont committed
894
			DST_RET(ISC_R_NOSPACE);
895
		isc_buffer_putuint8(data, (isc_uint8_t) e_bytes);
896
		isc_region_consume(&r, 1);
897
898
	} else {
		if (r.length < 3)
Francis Dupont's avatar
Francis Dupont committed
899
			DST_RET(ISC_R_NOSPACE);
900
901
		isc_buffer_putuint8(data, 0);
		isc_buffer_putuint16(data, (isc_uint16_t) e_bytes);
902
		isc_region_consume(&r, 3);
903
904
905
	}

	if (r.length < e_bytes + mod_bytes)
906
		DST_RET(ISC_R_NOSPACE);
907
908

	BN_bn2bin(rsa->e, r.base);
909
	isc_region_consume(&r, e_bytes);
910
911
912
913
	BN_bn2bin(rsa->n, r.base);

	isc_buffer_add(data, e_bytes + mod_bytes);

Francis Dupont's avatar
Francis Dupont committed
914
915
916
917
918
919
920
	ret = ISC_R_SUCCESS;
 err:
#if USE_EVP
	if (rsa != NULL)
		RSA_free(rsa);
#endif
	return (ret);
921
922
923
924
925
926
927
}

static isc_result_t
opensslrsa_fromdns(dst_key_t *key, isc_buffer_t *data) {
	RSA *rsa;
	isc_region_t r;
	unsigned int e_bytes;
Francis Dupont's avatar
Francis Dupont committed
928
929
930
#if USE_EVP
	EVP_PKEY *pkey;
#endif
931
932
933
934
935
936
937

	isc_buffer_remainingregion(data, &r);
	if (r.length == 0)
		return (ISC_R_SUCCESS);

	rsa = RSA_new();
	if (rsa == NULL)
938
		return (dst__openssl_toresult(ISC_R_NOMEMORY));
939
	SET_FLAGS(rsa);
940

Brian Wellington's avatar
Brian Wellington committed
941
942
	if (r.length < 1) {
		RSA_free(rsa);
943
		return (DST_R_INVALIDPUBLICKEY);
Brian Wellington's avatar
Brian Wellington committed
944
	}
945
946
947
948
	e_bytes = *r.base++;
	r.length--;

	if (e_bytes == 0) {
Brian Wellington's avatar
Brian Wellington committed
949
950
		if (r.length < 2) {
			RSA_free(rsa);
951
			return (DST_R_INVALIDPUBLICKEY);
Brian Wellington's avatar
Brian Wellington committed
952
		}
953
954
955
956
957
		e_bytes = ((*r.base++) << 8);
		e_bytes += *r.base++;
		r.length -= 2;
	}

Brian Wellington's avatar
Brian Wellington committed
958
959
	if (r.length < e_bytes) {
		RSA_free(rsa);
960
		return (DST_R_INVALIDPUBLICKEY);
Brian Wellington's avatar
Brian Wellington committed
961
	}
962
963
964
965
966
967
968
969
970
971
	rsa->e = BN_bin2bn(r.base, e_bytes, NULL);
	r.base += e_bytes;
	r.length -= e_bytes;

	rsa->n = BN_bin2bn(r.base, r.length, NULL);

	key->key_size = BN_num_bits(rsa->n);

	isc_buffer_forward(data, r.length);

Francis Dupont's avatar
Francis Dupont committed
972
973
974
975
976
977
978
979
980
981
982
983
984
985
#if USE_EVP
	pkey = EVP_PKEY_new();
	if (pkey == NULL) {
		RSA_free(rsa);
		return (ISC_R_NOMEMORY);
	}
	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
		EVP_PKEY_free(pkey);
		RSA_free(rsa);
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	}
	key->keydata.pkey = pkey;
	RSA_free(rsa);
#else
986
	key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
987
#endif
988
989
990
991
992
993

	return (ISC_R_SUCCESS);
}

static isc_result_t
opensslrsa_tofile(const dst_key_t *key, const char *directory) {
994
	int i;
995
996
	RSA *rsa;
	dst_private_t priv;
997
998
	unsigned char *bufs[8];
	isc_result_t result;
999

Francis Dupont's avatar
Francis Dupont committed
1000
1001
1002
1003
1004
1005
1006
#if USE_EVP
	if (key->keydata.pkey == NULL)
		return (DST_R_NULLKEY);
	rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
	if (rsa == NULL)
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
#else
1007
	if (key->keydata.rsa == NULL)
1008
		return (DST_R_NULLKEY);
1009
	rsa = key->keydata.rsa;
Francis Dupont's avatar
Francis Dupont committed
1010
#endif
1011

1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
	for (i = 0; i < 8; i++) {
		bufs[i] = isc_mem_get(key->mctx, BN_num_bytes(rsa->n));
		if (bufs[i] == NULL) {
			result = ISC_R_NOMEMORY;
			goto fail;
		}
	}

	i = 0;

	priv.elements[i].tag = TAG_RSA_MODULUS;
	priv.elements[i].length = BN_num_bytes(rsa->n);
	BN_bn2bin(rsa->n, bufs[i]);
	priv.elements[i].data = bufs[i];
	i++;

	priv.elements[i].tag = TAG_RSA_PUBLICEXPONENT;
	priv.elements[i].length = BN_num_bytes(rsa->e);
	BN_bn2bin(rsa->e, bufs[i]);
	priv.elements[i].data = bufs[i];
	i++;

Francis Dupont's avatar
Francis Dupont committed
1034
1035
1036
1037
1038
1039
1040
	if (rsa->d != NULL) {
		priv.elements[i].tag = TAG_RSA_PRIVATEEXPONENT;
		priv.elements[i].length = BN_num_bytes(rsa->d);
		BN_bn2bin(rsa->d, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1041

Francis Dupont's avatar
Francis Dupont committed
1042
1043
1044
1045
1046
1047
1048
	if (rsa->p != NULL) {
		priv.elements[i].tag = TAG_RSA_PRIME1;
		priv.elements[i].length = BN_num_bytes(rsa->p);
		BN_bn2bin(rsa->p, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1049

Francis Dupont's avatar
Francis Dupont committed
1050
1051
1052
1053
1054
1055
1056
	if (rsa->q != NULL) {
		priv.elements[i].tag = TAG_RSA_PRIME2;
		priv.elements[i].length = BN_num_bytes(rsa->q);
		BN_bn2bin(rsa->q, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1057

Francis Dupont's avatar
Francis Dupont committed
1058
1059
1060
1061
1062
1063
1064
	if (rsa->dmp1 != NULL) {
		priv.elements[i].tag = TAG_RSA_EXPONENT1;
		priv.elements[i].length = BN_num_bytes(rsa->dmp1);
		BN_bn2bin(rsa->dmp1, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1065

Francis Dupont's avatar
Francis Dupont committed
1066
1067
1068
1069
1070
1071
1072
	if (rsa->dmq1 != NULL) {
		priv.elements[i].tag = TAG_RSA_EXPONENT2;
		priv.elements[i].length = BN_num_bytes(rsa->dmq1);
		BN_bn2bin(rsa->dmq1, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
1073

Francis Dupont's avatar
Francis Dupont committed
1074
1075
1076
1077
1078
1079
1080
	if (rsa->iqmp != NULL) {
		priv.elements[i].tag = TAG_RSA_COEFFICIENT;
		priv.elements[i].length = BN_num_bytes(rsa->iqmp);
		BN_bn2bin(rsa->iqmp, bufs[i]);
		priv.elements[i].data = bufs[i];
		i++;
	}
Automatic Updater's avatar
Automatic Updater committed
1081

Francis Dupont's avatar
Francis Dupont committed
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
	if (key->engine != NULL) {
		priv.elements[i].tag = TAG_RSA_ENGINE;
		priv.elements[i].length = strlen(key->engine) + 1;
		priv.elements[i].data = (unsigned char *)key->engine;
		i++;
	}

	if (key->label != NULL) {
		priv.elements[i].tag = TAG_RSA_LABEL;
		priv.elements[i].length = strlen(key->label) + 1;
		priv.elements[i].data = (unsigned char *)key->label;
		i++;
	}
1095

1096

1097
	priv.nelements = i;
1098
	result = dst__privstruct_writefile(key, &priv, directory);
1099
 fail:
Francis Dupont's avatar
Francis Dupont committed
1100
1101
1102
#if USE_EVP
	RSA_free(rsa);
#endif
1103
1104
1105
1106
1107
	for (i = 0; i < 8; i++) {
		if (bufs[i] == NULL)
			break;
		isc_mem_put(key->mctx, bufs[i], BN_num_bytes(rsa->n));
	}
1108
	return (result);
1109
1110
1111
}

static isc_result_t
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
rsa_check(RSA *rsa, RSA *pub)
{
	/* Public parameters should be the same but if they are not set
	 * copy them from the public key. */
	if (pub != NULL) {
		if (rsa->n != NULL) {
			if (BN_cmp(rsa->n, pub->n) != 0)
				return (DST_R_INVALIDPRIVATEKEY);
		} else {
			rsa->n = pub->n;
			pub->n = NULL;
		}
		if (rsa->e != NULL) {
			if (BN_cmp(rsa->e, pub->e) != 0)
				return (DST_R_INVALIDPRIVATEKEY);
		} else {
			rsa->e = pub->e;
			pub->e = NULL;
		}
	}
	if (rsa->n == NULL || rsa->e == NULL)
		return (DST_R_INVALIDPRIVATEKEY);
	return (ISC_R_SUCCESS);
}

static isc_result_t
opensslrsa_parse(dst_key_t *key, isc_lex_t *lexer, dst_key_t *pub) {
1139
1140
1141
	dst_private_t priv;
	isc_result_t ret;
	int i;
1142
	RSA *rsa = NULL, *pubrsa = NULL;
1143
#ifdef USE_ENGINE
Francis Dupont's avatar
Francis Dupont committed
1144
	ENGINE *e = NULL;
1145
#endif
1146
	isc_mem_t *mctx = key->mctx;
Francis Dupont's avatar
Francis Dupont committed
1147
	const char *engine = NULL, *label = NULL;
1148
#if defined(USE_ENGINE) || USE_EVP
Francis Dupont's avatar
Francis Dupont committed
1149
	EVP_PKEY *pkey = NULL;
1150
#endif
1151

1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
#if USE_EVP
	if (pub != NULL && pub->keydata.pkey != NULL)
		pubrsa = EVP_PKEY_get1_RSA(pub->keydata.pkey);
#else
	if (pub != NULL && pub->keydata.rsa != NULL) {
		pubrsa = pub->keydata.rsa;
		pub->keydata.rsa = NULL;
	}
#endif

1162
	/* read private key file */
1163
	ret = dst__privstruct_parse(key, DST_ALG_RSA, lexer, mctx, &priv);
1164
1165
1166
	if (ret != ISC_R_SUCCESS)
		return (ret);

Francis Dupont's avatar
Francis Dupont committed
1167
1168
1169
	for (i = 0; i < priv.nelements; i++) {
		switch (priv.elements[i].tag) {
		case TAG_RSA_ENGINE:
Francis Dupont's avatar
Francis Dupont committed
1170
			engine = (char *)priv.elements[i].data;
Francis Dupont's avatar
Francis Dupont committed
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
			break;
		case TAG_RSA_LABEL:
			label = (char *)priv.elements[i].data;
			break;
		default:
			break;
		}
	}
	/*
	 * Is this key is stored in a HSM?
	 * See if we can fetch it.
	 */
Francis Dupont's avatar
Francis Dupont committed
1183
	if (label != NULL) {
1184
#ifdef USE_ENGINE
Francis Dupont's avatar
Francis Dupont committed
1185
1186
1187
		if (engine == NULL)
			DST_RET(DST_R_NOENGINE);
		e = dst__openssl_getengine(engine);
Francis Dupont's avatar
Francis Dupont committed
1188
1189
1190
1191
		if (e == NULL)
			DST_RET(DST_R_NOENGINE);
		pkey = ENGINE_load_private_key(e, label, NULL, NULL);
		if (pkey == NULL) {
1192
1193
			/* ERR_print_errors_fp(stderr); */
			DST_RET(ISC_R_NOTFOUND);
Francis Dupont's avatar
Francis Dupont committed
1194
		}
Francis Dupont's avatar
Francis Dupont committed
1195
		key->engine = isc_mem_strdup(key->mctx, engine);
Francis Dupont's avatar
Francis Dupont committed
1196
1197
1198
1199
1200
		if (key->engine == NULL)
			DST_RET(ISC_R_NOMEMORY);
		key->label = isc_mem_strdup(key->mctx, label);
		if (key->label == NULL)
			DST_RET(ISC_R_NOMEMORY);
1201
1202
1203
1204
1205
		rsa = EVP_PKEY_get1_RSA(pkey);
		if (rsa == NULL)
			DST_RET(dst__openssl_toresult(DST_R_OPENSSLFAILURE));
		if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
			DST_RET(DST_R_INVALIDPRIVATEKEY);
Francis Dupont's avatar
Francis Dupont committed
1206
1207
		if (pubrsa != NULL)
			RSA_free(pubrsa);
Francis Dupont's avatar
Francis Dupont committed
1208
1209
1210
		key->key_size = EVP_PKEY_bits(pkey);
#if USE_EVP
		key->keydata.pkey = pkey;
Francis Dupont's avatar
Francis Dupont committed
1211
		RSA_free(rsa);
Francis Dupont's avatar
Francis Dupont committed
1212
#else
1213
		key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
1214
1215
1216
		EVP_PKEY_free(pkey);
#endif
		dst__privstruct_free(&priv, mctx);
1217
		memset(&priv, 0, sizeof(priv));
Francis Dupont's avatar
Francis Dupont committed
1218
		return (ISC_R_SUCCESS);
1219
1220
1221
#else
		DST_RET(DST_R_NOENGINE);
#endif
Automatic Updater's avatar
Automatic Updater committed
1222
	}
Francis Dupont's avatar
Francis Dupont committed
1223

1224
1225
1226
	rsa = RSA_new();
	if (rsa == NULL)
		DST_RET(ISC_R_NOMEMORY);
1227
	SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
1228
1229
1230
1231
1232

#if USE_EVP
	pkey = EVP_PKEY_new();
	if (pkey == NULL)
		DST_RET(ISC_R_NOMEMORY);
Francis Dupont's avatar
Francis Dupont committed
1233
	if (!EVP_PKEY_set1_RSA(pkey, rsa))
Francis Dupont's avatar
Francis Dupont committed
1234
1235
1236
		DST_RET(ISC_R_FAILURE);
	key->keydata.pkey = pkey;
#else
1237
	key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
1238
#endif
1239
1240
1241

	for (i = 0; i < priv.nelements; i++) {
		BIGNUM *bn;
Francis Dupont's avatar
Francis Dupont committed
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
		switch (priv.elements[i].tag) {
		case TAG_RSA_ENGINE:
			continue;
		case TAG_RSA_LABEL:
			continue;
		case TAG_RSA_PIN:
			continue;
		default:
			bn = BN_bin2bn(priv.elements[i].data,
				       priv.elements[i].length, NULL);
			if (bn == NULL)
				DST_RET(ISC_R_NOMEMORY);
Automatic Updater's avatar
Automatic Updater committed
1254
		}
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280

		switch (priv.elements[i].tag) {
			case TAG_RSA_MODULUS:
				rsa->n = bn;
				break;
			case TAG_RSA_PUBLICEXPONENT:
				rsa->e = bn;
				break;
			case TAG_RSA_PRIVATEEXPONENT:
				rsa->d = bn;
				break;
			case TAG_RSA_PRIME1:
				rsa->p = bn;
				break;
			case TAG_RSA_PRIME2:
				rsa->q = bn;
				break;
			case TAG_RSA_EXPONENT1:
				rsa->dmp1 = bn;
				break;
			case TAG_RSA_EXPONENT2:
				rsa->dmq1 = bn;
				break;
			case TAG_RSA_COEFFICIENT:
				rsa->iqmp = bn;
				break;
Brian Wellington's avatar
Brian Wellington committed
1281
		}
1282
1283
	}
	dst__privstruct_free(&priv, mctx);
1284
	memset(&priv, 0, sizeof(priv));
1285

1286
1287
	if (rsa_check(rsa, pubrsa) != ISC_R_SUCCESS)
		DST_RET(DST_R_INVALIDPRIVATEKEY);
1288
	key->key_size = BN_num_bits(rsa->n);
1289
1290
	if (pubrsa != NULL)
		RSA_free(pubrsa);
Francis Dupont's avatar
Francis Dupont committed
1291
1292
1293
#if USE_EVP
	RSA_free(rsa);
#endif
1294
1295
1296
1297

	return (ISC_R_SUCCESS);

 err:
Francis Dupont's avatar
Francis Dupont committed
1298
1299
1300