opensslrsa_link.c 39.6 KB
Newer Older
1
/*
2
 * Copyright (C) 2000-2009, 2011-2017  Internet Systems Consortium, Inc. ("ISC")
3
 *
4 5 6
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
7 8 9 10
 */

/*
 * Principal Author: Brian Wellington
Mark Andrews's avatar
Mark Andrews committed
11
 * $Id$
12
 */
13
#ifdef OPENSSL
14 15
#include <config.h>

Francis Dupont's avatar
Francis Dupont committed
16
#ifndef USE_EVP
17 18 19
#if !defined(HAVE_EVP_SHA256) || !defined(HAVE_EVP_SHA512)
#define USE_EVP 0
#else
Francis Dupont's avatar
Francis Dupont committed
20 21
#define USE_EVP 1
#endif
22
#endif
23 24 25 26


#include <isc/entropy.h>
#include <isc/md5.h>
Mark Andrews's avatar
Mark Andrews committed
27 28
#include <isc/mem.h>
#include <isc/safe.h>
29
#include <isc/sha1.h>
30
#include <isc/sha2.h>
31 32 33
#include <isc/string.h>
#include <isc/util.h>

34 35
#include <pk11/site.h>

36 37 38
#include <dst/result.h>

#include "dst_internal.h"
39
#include "dst_openssl.h"
40 41 42 43 44
#include "dst_parse.h"

#include <openssl/err.h>
#include <openssl/objects.h>
#include <openssl/rsa.h>
45 46 47
#if OPENSSL_VERSION_NUMBER > 0x00908000L
#include <openssl/bn.h>
#endif
48
#ifdef USE_ENGINE
Francis Dupont's avatar
Francis Dupont committed
49
#include <openssl/engine.h>
50
#endif
51

52 53 54 55 56 57 58
/*
 * Limit the size of public exponents.
 */
#ifndef RSA_MAX_PUBEXP_BITS
#define RSA_MAX_PUBEXP_BITS    35
#endif

59 60 61 62
/*
 * We don't use configure for windows so enforce the OpenSSL version
 * here.  Unlike with configure we don't support overriding this test.
 */
Mark Andrews's avatar
Mark Andrews committed
63
#ifdef WIN32
64
#if !((OPENSSL_VERSION_NUMBER >= 0x009070cfL && \
65
       OPENSSL_VERSION_NUMBER < 0x00908000L) || \
66 67 68
      (OPENSSL_VERSION_NUMBER >= 0x0090804fL && \
       OPENSSL_VERSION_NUMBER < 0x10002000L) || \
      OPENSSL_VERSION_NUMBER >= 0x1000205fL)
69 70 71 72 73
#error Please upgrade OpenSSL to 0.9.8d/0.9.7l or greater.
#endif
#endif


74
	/*
Francis Dupont's avatar
Francis Dupont committed
75 76
	 * XXXMPA  Temporarily disable RSA_BLINDING as it requires
	 * good quality random data that cannot currently be guaranteed.
77 78 79 80 81
	 * XXXMPA  Find which versions of openssl use pseudo random data
	 * and set RSA_FLAG_BLINDING for those.
	 */

#if 0
82 83
#if OPENSSL_VERSION_NUMBER < 0x0090601fL
#define SET_FLAGS(rsa) \
84 85 86 87
	do { \
	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
	(rsa)->flags |= RSA_FLAG_BLINDING; \
	} while (0)
88
#else
89 90 91 92
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags |= RSA_FLAG_BLINDING; \
	} while (0)
93
#endif
94 95 96 97 98 99 100 101
#endif

#if OPENSSL_VERSION_NUMBER < 0x0090601fL
#define SET_FLAGS(rsa) \
	do { \
	(rsa)->flags &= ~(RSA_FLAG_CACHE_PUBLIC | RSA_FLAG_CACHE_PRIVATE); \
	(rsa)->flags &= ~RSA_FLAG_BLINDING; \
	} while (0)
102
#elif OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
103
#if defined(RSA_FLAG_NO_BLINDING)
104 105 106 107 108
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
		(rsa)->flags |= RSA_FLAG_NO_BLINDING; \
	} while (0)
109 110 111 112 113 114
#else
#define SET_FLAGS(rsa) \
	do { \
		(rsa)->flags &= ~RSA_FLAG_BLINDING; \
	} while (0)
#endif
115 116 117 118 119 120 121
#else
#define SET_FLAGS(rsa) \
	do { \
		RSA_clear_flags(rsa, RSA_FLAG_BLINDING); \
		RSA_set_flags(rsa, RSA_FLAG_NO_BLINDING); \
	} while (0)
#endif
Francis Dupont's avatar
Francis Dupont committed
122 123
#define DST_RET(a) {ret = a; goto err;}

124
#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240
/* From OpenSSL 1.1.0 */
static int
RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) {

	/*
	 * If the fields n and e in r are NULL, the corresponding input
	 * parameters MUST be non-NULL for n and e.  d may be
	 * left NULL (in case only the public key is used).
	 */
	if ((r->n == NULL && n == NULL) || (r->e == NULL && e == NULL))
		return 0;

	if (n != NULL) {
		BN_free(r->n);
		r->n = n;
	}
	if (e != NULL) {
		BN_free(r->e);
		r->e = e;
	}
	if (d != NULL) {
		BN_free(r->d);
		r->d = d;
	}

	return 1;
}

static int
RSA_set0_factors(RSA *r, BIGNUM *p, BIGNUM *q) {

	/*
	 * If the fields p and q in r are NULL, the corresponding input
	 * parameters MUST be non-NULL.
	 */
	if ((r->p == NULL && p == NULL) || (r->q == NULL && q == NULL))
		return 0;

	if (p != NULL) {
		BN_free(r->p);
		r->p = p;
	}
	if (q != NULL) {
		BN_free(r->q);
		r->q = q;
	}

	return 1;
}

static int
RSA_set0_crt_params(RSA *r, BIGNUM *dmp1, BIGNUM *dmq1, BIGNUM *iqmp) {
	/*
	 * If the fields dmp1, dmq1 and iqmp in r are NULL, the
	 * corresponding input parameters MUST be non-NULL.
	 */
	if ((r->dmp1 == NULL && dmp1 == NULL) ||
	    (r->dmq1 == NULL && dmq1 == NULL) ||
	    (r->iqmp == NULL && iqmp == NULL))
		return 0;

	if (dmp1 != NULL) {
		BN_free(r->dmp1);
		r->dmp1 = dmp1;
	}
	if (dmq1 != NULL) {
		BN_free(r->dmq1);
		r->dmq1 = dmq1;
	}
	if (iqmp != NULL) {
		BN_free(r->iqmp);
		r->iqmp = iqmp;
	}

	return 1;
}

static void
RSA_get0_key(const RSA *r,
	     const BIGNUM **n, const BIGNUM **e, const BIGNUM **d)
{
	if (n != NULL)
		*n = r->n;
	if (e != NULL)
		*e = r->e;
	if (d != NULL)
		*d = r->d;
}

static void
RSA_get0_factors(const RSA *r, const BIGNUM **p, const BIGNUM **q) {
	if (p != NULL)
		*p = r->p;
	if (q != NULL)
	*q = r->q;
}

static void
RSA_get0_crt_params(const RSA *r, const BIGNUM **dmp1, const BIGNUM **dmq1,
		    const BIGNUM **iqmp)
{
	if (dmp1 != NULL)
		*dmp1 = r->dmp1;
	if (dmq1 != NULL)
		*dmq1 = r->dmq1;
	if (iqmp != NULL)
		*iqmp = r->iqmp;
}

static int
RSA_test_flags(const RSA *r, int flags) {
	return (r->flags & flags);
}

#endif

241 242 243 244
static isc_result_t opensslrsa_todns(const dst_key_t *key, isc_buffer_t *data);

static isc_result_t
opensslrsa_createctx(dst_key_t *key, dst_context_t *dctx) {
Mark Andrews's avatar
Mark Andrews committed
245 246
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx;
247
	const EVP_MD *type = NULL;
Mark Andrews's avatar
Mark Andrews committed
248 249
#endif

250
	UNUSED(key);
251
#ifndef PK11_MD5_DISABLE
252
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
253
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
254 255 256
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
257 258 259 260 261 262
#else
	REQUIRE(dctx->key->key_alg == DST_ALG_RSASHA1 ||
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
#endif
263

264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290
	/*
	 * Reject incorrect RSA key lengths.
	 */
	switch (dctx->key->key_alg) {
	case DST_ALG_RSAMD5:
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		/* From RFC 3110 */
		if (dctx->key->key_size > 4096)
			return (ISC_R_FAILURE);
		break;
	case DST_ALG_RSASHA256:
		/* From RFC 5702 */
		if ((dctx->key->key_size < 512) ||
		    (dctx->key->key_size > 4096))
			return (ISC_R_FAILURE);
		break;
	case DST_ALG_RSASHA512:
		/* From RFC 5702 */
		if ((dctx->key->key_size < 1024) ||
		    (dctx->key->key_size > 4096))
			return (ISC_R_FAILURE);
		break;
	default:
		INSIST(0);
	}

Francis Dupont's avatar
Francis Dupont committed
291 292 293 294 295
#if USE_EVP
	evp_md_ctx = EVP_MD_CTX_create();
	if (evp_md_ctx == NULL)
		return (ISC_R_NOMEMORY);

296
	switch (dctx->key->key_alg) {
297
#ifndef PK11_MD5_DISABLE
298
	case DST_ALG_RSAMD5:
Francis Dupont's avatar
Francis Dupont committed
299
		type = EVP_md5();	/* MD5 + RSA */
300
		break;
301
#endif
302 303
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
Francis Dupont's avatar
Francis Dupont committed
304
		type = EVP_sha1();	/* SHA1 + RSA */
305
		break;
306
#ifdef HAVE_EVP_SHA256
307 308 309
	case DST_ALG_RSASHA256:
		type = EVP_sha256();	/* SHA256 + RSA */
		break;
310 311
#endif
#ifdef HAVE_EVP_SHA512
312 313 314
	case DST_ALG_RSASHA512:
		type = EVP_sha512();
		break;
315
#endif
316 317 318
	default:
		INSIST(0);
	}
Francis Dupont's avatar
Francis Dupont committed
319 320 321

	if (!EVP_DigestInit_ex(evp_md_ctx, type, NULL)) {
		EVP_MD_CTX_destroy(evp_md_ctx);
322 323
		return (dst__openssl_toresult3(dctx->category,
					       "EVP_DigestInit_ex",
324
					       ISC_R_FAILURE));
Francis Dupont's avatar
Francis Dupont committed
325 326 327
	}
	dctx->ctxdata.evp_md_ctx = evp_md_ctx;
#else
328
	switch (dctx->key->key_alg) {
329
#ifndef PK11_MD5_DISABLE
330 331 332 333 334 335 336 337 338 339 340
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx;

			md5ctx = isc_mem_get(dctx->mctx, sizeof(isc_md5_t));
			if (md5ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_md5_init(md5ctx);
			dctx->ctxdata.md5ctx = md5ctx;
		}
		break;
341
#endif
342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx;

			sha1ctx = isc_mem_get(dctx->mctx, sizeof(isc_sha1_t));
			if (sha1ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha1_init(sha1ctx);
			dctx->ctxdata.sha1ctx = sha1ctx;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx;

Automatic Updater's avatar
Automatic Updater committed
358
			sha256ctx = isc_mem_get(dctx->mctx,
359 360 361 362 363 364 365 366 367 368 369
						sizeof(isc_sha256_t));
			if (sha256ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha256_init(sha256ctx);
			dctx->ctxdata.sha256ctx = sha256ctx;
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx;

Automatic Updater's avatar
Automatic Updater committed
370
			sha512ctx = isc_mem_get(dctx->mctx,
371 372 373 374 375 376 377 378 379
						sizeof(isc_sha512_t));
			if (sha512ctx == NULL)
				return (ISC_R_NOMEMORY);
			isc_sha512_init(sha512ctx);
			dctx->ctxdata.sha512ctx = sha512ctx;
		}
		break;
	default:
		INSIST(0);
380
	}
Francis Dupont's avatar
Francis Dupont committed
381
#endif
382 383 384 385 386 387

	return (ISC_R_SUCCESS);
}

static void
opensslrsa_destroyctx(dst_context_t *dctx) {
Mark Andrews's avatar
Mark Andrews committed
388 389 390 391
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
#endif

392
#ifndef PK11_MD5_DISABLE
393
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
394
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
395 396 397
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
398 399 400 401 402 403
#else
	REQUIRE(dctx->key->key_alg == DST_ALG_RSASHA1 ||
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
#endif
404

Francis Dupont's avatar
Francis Dupont committed
405 406 407 408 409 410
#if USE_EVP
	if (evp_md_ctx != NULL) {
		EVP_MD_CTX_destroy(evp_md_ctx);
		dctx->ctxdata.evp_md_ctx = NULL;
	}
#else
411
	switch (dctx->key->key_alg) {
412
#ifndef PK11_MD5_DISABLE
413 414 415 416 417 418 419 420 421 422
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			if (md5ctx != NULL) {
				isc_md5_invalidate(md5ctx);
				isc_mem_put(dctx->mctx, md5ctx,
					    sizeof(isc_md5_t));
				dctx->ctxdata.md5ctx = NULL;
			}
423
		}
424
		break;
425
#endif
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			if (sha1ctx != NULL) {
				isc_sha1_invalidate(sha1ctx);
				isc_mem_put(dctx->mctx, sha1ctx,
					    sizeof(isc_sha1_t));
				dctx->ctxdata.sha1ctx = NULL;
			}
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			if (sha256ctx != NULL) {
				isc_sha256_invalidate(sha256ctx);
				isc_mem_put(dctx->mctx, sha256ctx,
					    sizeof(isc_sha256_t));
				dctx->ctxdata.sha256ctx = NULL;
			}
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			if (sha512ctx != NULL) {
				isc_sha512_invalidate(sha512ctx);
				isc_mem_put(dctx->mctx, sha512ctx,
					    sizeof(isc_sha512_t));
				dctx->ctxdata.sha512ctx = NULL;
			}
461
		}
462 463 464
		break;
	default:
		INSIST(0);
465
	}
Francis Dupont's avatar
Francis Dupont committed
466
#endif
467 468 469 470
}

static isc_result_t
opensslrsa_adddata(dst_context_t *dctx, const isc_region_t *data) {
Mark Andrews's avatar
Mark Andrews committed
471 472
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
Automatic Updater's avatar
Automatic Updater committed
473
#endif
Mark Andrews's avatar
Mark Andrews committed
474

475
#ifndef PK11_MD5_DISABLE
476
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
477
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
478 479 480
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
481 482 483 484 485 486
#else
	REQUIRE(dctx->key->key_alg == DST_ALG_RSASHA1 ||
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
#endif
487

Francis Dupont's avatar
Francis Dupont committed
488 489
#if USE_EVP
	if (!EVP_DigestUpdate(evp_md_ctx, data->base, data->length)) {
490 491
		return (dst__openssl_toresult3(dctx->category,
					       "EVP_DigestUpdate",
492
					       ISC_R_FAILURE));
Francis Dupont's avatar
Francis Dupont committed
493 494
	}
#else
495
	switch (dctx->key->key_alg) {
496
#ifndef PK11_MD5_DISABLE
497 498 499 500 501 502 503
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_update(md5ctx, data->base, data->length);
		}
		break;
504
#endif
505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_update(sha1ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_update(sha256ctx, data->base, data->length);
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_update(sha512ctx, data->base, data->length);
		}
		break;
	default:
		INSIST(0);
529
	}
Francis Dupont's avatar
Francis Dupont committed
530
#endif
531 532 533
	return (ISC_R_SUCCESS);
}

534 535
#if ! USE_EVP && OPENSSL_VERSION_NUMBER < 0x00908000L
/*
Mark Andrews's avatar
RFC5702  
Mark Andrews committed
536
 * Digest prefixes from RFC 5702.
537 538 539 540 541 542 543 544 545 546 547 548
 */
static unsigned char sha256_prefix[] =
	 { 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
	   0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20};
static unsigned char sha512_prefix[] =
	 { 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48,
	   0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04, 0x40};
#define PREFIXLEN sizeof(sha512_prefix)
#else
#define PREFIXLEN 0
#endif

549 550 551 552
static isc_result_t
opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
	dst_key_t *key = dctx->key;
	isc_region_t r;
Francis Dupont's avatar
Francis Dupont committed
553 554 555 556 557 558
	unsigned int siglen = 0;
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
	EVP_PKEY *pkey = key->keydata.pkey;
#else
	RSA *rsa = key->keydata.rsa;
559
	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
560
	unsigned char digest[PREFIXLEN + ISC_SHA512_DIGESTLENGTH];
561
	int status;
562 563
	int type = 0;
	unsigned int digestlen = 0;
564 565 566 567
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	unsigned int prefixlen = 0;
	const unsigned char *prefix = NULL;
#endif
Francis Dupont's avatar
Francis Dupont committed
568
#endif
569

570
#ifndef PK11_MD5_DISABLE
571
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
572
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
573 574 575
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
576 577 578 579 580 581
#else
	REQUIRE(dctx->key->key_alg == DST_ALG_RSASHA1 ||
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
#endif
582

583 584
	isc_buffer_availableregion(sig, &r);

Francis Dupont's avatar
Francis Dupont committed
585 586 587 588 589
#if USE_EVP
	if (r.length < (unsigned int) EVP_PKEY_size(pkey))
		return (ISC_R_NOSPACE);

	if (!EVP_SignFinal(evp_md_ctx, r.base, &siglen, pkey)) {
590 591
		return (dst__openssl_toresult3(dctx->category,
					       "EVP_SignFinal",
592
					       ISC_R_FAILURE));
Francis Dupont's avatar
Francis Dupont committed
593 594
	}
#else
595 596 597
	if (r.length < (unsigned int) RSA_size(rsa))
		return (ISC_R_NOSPACE);

598
	switch (dctx->key->key_alg) {
599
#ifndef PK11_MD5_DISABLE
600 601 602 603 604 605 606 607 608
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_final(md5ctx, digest);
			type = NID_md5;
			digestlen = ISC_MD5_DIGESTLENGTH;
		}
		break;
609
#endif
610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_final(sha1ctx, digest);
			type = NID_sha1;
			digestlen = ISC_SHA1_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_final(digest, sha256ctx);
			digestlen = ISC_SHA256_DIGESTLENGTH;
626 627 628 629 630 631
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha256_prefix;
			prefixlen = sizeof(sha256_prefix);
#else
			type = NID_sha256;
#endif
632 633 634 635 636 637 638 639
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_final(digest, sha512ctx);
			digestlen = ISC_SHA512_DIGESTLENGTH;
640 641 642 643 644 645
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha512_prefix;
			prefixlen = sizeof(sha512_prefix);
#else
			type = NID_sha512;
#endif
646 647 648 649
		}
		break;
	default:
		INSIST(0);
650 651
	}

652 653
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	switch (dctx->key->key_alg) {
654
#ifndef PK11_MD5_DISABLE
655
	case DST_ALG_RSAMD5:
656
#endif
657 658 659 660 661 662 663 664 665 666 667 668 669 670
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		INSIST(type != 0);
		status = RSA_sign(type, digest, digestlen, r.base,
				  &siglen, rsa);
		break;

	case DST_ALG_RSASHA256:
	case DST_ALG_RSASHA512:
		INSIST(prefix != NULL);
		INSIST(prefixlen != 0);
		INSIST(prefixlen + digestlen <= sizeof(digest));

		memmove(digest + prefixlen, digest, digestlen);
671
		memmove(digest, prefix, prefixlen);
672 673 674 675 676 677 678 679 680 681 682 683 684 685
		status = RSA_private_encrypt(digestlen + prefixlen,
					     digest, r.base, rsa,
					     RSA_PKCS1_PADDING);
		if (status < 0)
			status = 0;
		else
			siglen = status;
		break;

	default:
		INSIST(0);
	}
#else
	INSIST(type != 0);
686
	status = RSA_sign(type, digest, digestlen, r.base, &siglen, rsa);
687
#endif
688
	if (status == 0)
689 690
		return (dst__openssl_toresult3(dctx->category,
					       "RSA_sign",
691
					       DST_R_OPENSSLFAILURE));
Francis Dupont's avatar
Francis Dupont committed
692
#endif
693 694 695 696 697 698 699

	isc_buffer_add(sig, siglen);

	return (ISC_R_SUCCESS);
}

static isc_result_t
700
opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
701
	dst_key_t *key = dctx->key;
Francis Dupont's avatar
Francis Dupont committed
702
	int status = 0;
703
	const BIGNUM *e = NULL;
Francis Dupont's avatar
Francis Dupont committed
704 705 706
#if USE_EVP
	EVP_MD_CTX *evp_md_ctx = dctx->ctxdata.evp_md_ctx;
	EVP_PKEY *pkey = key->keydata.pkey;
707 708
	RSA *rsa;
	int bits;
Francis Dupont's avatar
Francis Dupont committed
709
#else
710 711 712 713
	/* note: ISC_SHA512_DIGESTLENGTH >= ISC_*_DIGESTLENGTH */
	unsigned char digest[ISC_SHA512_DIGESTLENGTH];
	int type = 0;
	unsigned int digestlen = 0;
Francis Dupont's avatar
Francis Dupont committed
714
	RSA *rsa = key->keydata.rsa;
715 716 717 718
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	unsigned int prefixlen = 0;
	const unsigned char *prefix = NULL;
#endif
Francis Dupont's avatar
Francis Dupont committed
719
#endif
720

721
#ifndef PK11_MD5_DISABLE
722
	REQUIRE(dctx->key->key_alg == DST_ALG_RSAMD5 ||
723
		dctx->key->key_alg == DST_ALG_RSASHA1 ||
724 725 726
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
727 728 729 730 731 732
#else
	REQUIRE(dctx->key->key_alg == DST_ALG_RSASHA1 ||
		dctx->key->key_alg == DST_ALG_NSEC3RSASHA1 ||
		dctx->key->key_alg == DST_ALG_RSASHA256 ||
		dctx->key->key_alg == DST_ALG_RSASHA512);
#endif
733

Francis Dupont's avatar
Francis Dupont committed
734
#if USE_EVP
735 736 737
	rsa = EVP_PKEY_get1_RSA(pkey);
	if (rsa == NULL)
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
738 739
	RSA_get0_key(rsa, NULL, &e, NULL);
	bits = BN_num_bits(e);
740 741 742 743
	RSA_free(rsa);
	if (bits > maxbits && maxbits != 0)
		return (DST_R_VERIFYFAILURE);

Francis Dupont's avatar
Francis Dupont committed
744
	status = EVP_VerifyFinal(evp_md_ctx, sig->base, sig->length, pkey);
745 746 747 748 749 750 751 752 753 754
	switch (status) {
	case 1:
		return (ISC_R_SUCCESS);
	case 0:
		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
	default:
		return (dst__openssl_toresult3(dctx->category,
					       "EVP_VerifyFinal",
					       DST_R_VERIFYFAILURE));
	}
Francis Dupont's avatar
Francis Dupont committed
755
#else
756 757
	RSA_get0_key(rsa, NULL, &e, NULL);
	if (BN_num_bits(e) > maxbits && maxbits != 0)
758 759
		return (DST_R_VERIFYFAILURE);

760
	switch (dctx->key->key_alg) {
761
#ifndef PK11_MD5_DISABLE
762 763 764 765 766 767 768 769 770
	case DST_ALG_RSAMD5:
		{
			isc_md5_t *md5ctx = dctx->ctxdata.md5ctx;

			isc_md5_final(md5ctx, digest);
			type = NID_md5;
			digestlen = ISC_MD5_DIGESTLENGTH;
		}
		break;
771
#endif
772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		{
			isc_sha1_t *sha1ctx = dctx->ctxdata.sha1ctx;

			isc_sha1_final(sha1ctx, digest);
			type = NID_sha1;
			digestlen = ISC_SHA1_DIGESTLENGTH;
		}
		break;
	case DST_ALG_RSASHA256:
		{
			isc_sha256_t *sha256ctx = dctx->ctxdata.sha256ctx;

			isc_sha256_final(digest, sha256ctx);
			digestlen = ISC_SHA256_DIGESTLENGTH;
788 789 790 791 792 793
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha256_prefix;
			prefixlen = sizeof(sha256_prefix);
#else
			type = NID_sha256;
#endif
794 795 796 797 798 799 800 801
		}
		break;
	case DST_ALG_RSASHA512:
		{
			isc_sha512_t *sha512ctx = dctx->ctxdata.sha512ctx;

			isc_sha512_final(digest, sha512ctx);
			digestlen = ISC_SHA512_DIGESTLENGTH;
802 803 804 805 806 807
#if OPENSSL_VERSION_NUMBER < 0x00908000L
			prefix = sha512_prefix;
			prefixlen = sizeof(sha512_prefix);
#else
			type = NID_sha512;
#endif
808 809 810 811
		}
		break;
	default:
		INSIST(0);
812 813
	}

814
	if (sig->length != (unsigned int) RSA_size(rsa))
815 816
		return (DST_R_VERIFYFAILURE);

817 818
#if OPENSSL_VERSION_NUMBER < 0x00908000L
	switch (dctx->key->key_alg) {
819
#ifndef PK11_MD5_DISABLE
820
	case DST_ALG_RSAMD5:
821
#endif
822 823 824 825 826 827 828
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		INSIST(type != 0);
		status = RSA_verify(type, digest, digestlen, sig->base,
				    RSA_size(rsa), rsa);
		break;

Automatic Updater's avatar
Automatic Updater committed
829
	case DST_ALG_RSASHA256:
830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846
	case DST_ALG_RSASHA512:
		{
			/*
			 * 1024 is big enough for all valid RSA bit sizes
			 * for use with DNSSEC.
			 */
			unsigned char original[PREFIXLEN + 1024];

			INSIST(prefix != NULL);
			INSIST(prefixlen != 0U);

			if (RSA_size(rsa) > (int)sizeof(original))
				return (DST_R_VERIFYFAILURE);

			status = RSA_public_decrypt(sig->length, sig->base,
						    original, rsa,
						    RSA_PKCS1_PADDING);
Automatic Updater's avatar
Automatic Updater committed
847
			if (status <= 0)
848 849
				return (dst__openssl_toresult3(
						dctx->category,
850 851
						"RSA_public_decrypt",
						DST_R_VERIFYFAILURE));
Automatic Updater's avatar
Automatic Updater committed
852
			if (status != (int)(prefixlen + digestlen))
853
				return (DST_R_VERIFYFAILURE);
854
			if (!isc_safe_memequal(original, prefix, prefixlen))
855
				return (DST_R_VERIFYFAILURE);
856 857
			if (!isc_safe_memequal(original + prefixlen,
					    digest, digestlen))
858 859 860 861 862 863 864 865 866 867
				return (DST_R_VERIFYFAILURE);
			status = 1;
		}
		break;

	default:
		INSIST(0);
	}
#else
	INSIST(type != 0);
868
	status = RSA_verify(type, digest, digestlen, sig->base,
869
			     RSA_size(rsa), rsa);
Francis Dupont's avatar
Francis Dupont committed
870
#endif
Francis Dupont's avatar
Francis Dupont committed
871
	if (status != 1)
872
		return (dst__openssl_toresult(DST_R_VERIFYFAILURE));
873
	return (ISC_R_SUCCESS);
874
#endif
875 876
}

877 878 879 880 881
static isc_result_t
opensslrsa_verify(dst_context_t *dctx, const isc_region_t *sig) {
	return (opensslrsa_verify2(dctx, 0, sig));
}

882 883 884
static isc_boolean_t
opensslrsa_compare(const dst_key_t *key1, const dst_key_t *key2) {
	int status;
Francis Dupont's avatar
Francis Dupont committed
885
	RSA *rsa1 = NULL, *rsa2 = NULL;
886 887 888 889 890
	const BIGNUM *n1 = NULL, *n2 = NULL;
	const BIGNUM *e1 = NULL, *e2 = NULL;
	const BIGNUM *d1 = NULL, *d2 = NULL;
	const BIGNUM *p1 = NULL, *p2 = NULL;
	const BIGNUM *q1 = NULL, *q2 = NULL;
Francis Dupont's avatar
Francis Dupont committed
891 892 893
#if USE_EVP
	EVP_PKEY *pkey1, *pkey2;
#endif
894

Francis Dupont's avatar
Francis Dupont committed
895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910
#if USE_EVP
	pkey1 = key1->keydata.pkey;
	pkey2 = key2->keydata.pkey;
	/*
	 * The pkey reference will keep these around after
	 * the RSA_free() call.
	 */
	if (pkey1 != NULL) {
		rsa1 = EVP_PKEY_get1_RSA(pkey1);
		RSA_free(rsa1);
	}
	if (pkey2 != NULL) {
		rsa2 = EVP_PKEY_get1_RSA(pkey2);
		RSA_free(rsa2);
	}
#else
911 912
	rsa1 = key1->keydata.rsa;
	rsa2 = key2->keydata.rsa;
Francis Dupont's avatar
Francis Dupont committed
913
#endif
914 915 916 917 918 919

	if (rsa1 == NULL && rsa2 == NULL)
		return (ISC_TRUE);
	else if (rsa1 == NULL || rsa2 == NULL)
		return (ISC_FALSE);

920 921 922
	RSA_get0_key(rsa1, &n1, &e1, &d1);
	RSA_get0_key(rsa2, &n2, &e2, &d2);
	status = BN_cmp(n1, n2) || BN_cmp(e1, e2);
923 924 925 926

	if (status != 0)
		return (ISC_FALSE);

Francis Dupont's avatar
Francis Dupont committed
927
#if USE_EVP
928 929 930 931
	if (RSA_test_flags(rsa1, RSA_FLAG_EXT_PKEY) != 0 ||
	    RSA_test_flags(rsa2, RSA_FLAG_EXT_PKEY) != 0) {
		if (RSA_test_flags(rsa1, RSA_FLAG_EXT_PKEY) == 0 ||
		    RSA_test_flags(rsa2, RSA_FLAG_EXT_PKEY) == 0)
Francis Dupont's avatar
Francis Dupont committed
932 933 934 935 936 937 938 939
			return (ISC_FALSE);
		/*
		 * Can't compare private parameters, BTW does it make sense?
		 */
		return (ISC_TRUE);
	}
#endif

940 941
	if (d1 != NULL || d2 != NULL) {
		if (d1 == NULL || d2 == NULL)
942
			return (ISC_FALSE);
943 944 945
		RSA_get0_factors(rsa1, &p1, &q1);
		RSA_get0_factors(rsa2, &p2, &q2);
		status = BN_cmp(d1, d2) || BN_cmp(p1, p1) || BN_cmp(q1, q2);
946 947 948 949 950 951 952

		if (status != 0)
			return (ISC_FALSE);
	}
	return (ISC_TRUE);
}

953 954
#if OPENSSL_VERSION_NUMBER > 0x00908000L
static int
Evan Hunt's avatar
Evan Hunt committed
955
progress_cb(int p, int n, BN_GENCB *cb) {
956 957 958 959
	union {
		void *dptr;
		void (*fptr)(int);
	} u;
960 961

	UNUSED(n);
962

963
	u.dptr = BN_GENCB_get_arg(cb);
964 965
	if (u.fptr != NULL)
		u.fptr(p);
966 967 968 969
	return (1);
}
#endif

970
static isc_result_t
971
opensslrsa_generate(dst_key_t *key, int exp, void (*callback)(int)) {
972
#if OPENSSL_VERSION_NUMBER > 0x00908000L
Mark Andrews's avatar
Mark Andrews committed
973
	isc_result_t ret = DST_R_OPENSSLFAILURE;
974 975 976 977
	union {
		void *dptr;
		void (*fptr)(int);
	} u;
978 979
	RSA *rsa = RSA_new();
	BIGNUM *e = BN_new();
980
#if OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER)
981 982 983
	BN_GENCB _cb;
#endif
	BN_GENCB *cb = BN_GENCB_new();
Francis Dupont's avatar
Francis Dupont committed
984 985 986
#if USE_EVP
	EVP_PKEY *pkey = EVP_PKEY_new();
#endif
987

988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014
	/*
	 * Reject incorrect RSA key lengths.
	 */
	switch (key->key_alg) {
	case DST_ALG_RSAMD5:
	case DST_ALG_RSASHA1:
	case DST_ALG_NSEC3RSASHA1:
		/* From RFC 3110 */
		if (key->key_size > 4096)
			goto err;
		break;
	case DST_ALG_RSASHA256:
		/* From RFC 5702 */
		if ((key->key_size < 512) ||
		    (key->key_size > 4096))
			goto err;
		break;
	case DST_ALG_RSASHA512:
		/* From RFC 5702 */
		if ((key->key_size < 1024) ||
		    (key->key_size > 4096))
			goto err;
		break;
	default:
		INSIST(0);
	}

1015
	if (rsa == NULL || e == NULL || cb == NULL)
1016
		goto err;
Francis Dupont's avatar
Francis Dupont committed
1017 1018 1019 1020 1021 1022
#if USE_EVP
	if (pkey == NULL)
		goto err;
	if (!EVP_PKEY_set1_RSA(pkey, rsa))
		goto err;
#endif
1023

1024 1025 1026 1027 1028
	if (exp == 0) {
		/* RSA_F4 0x10001 */
		BN_set_bit(e, 0);
		BN_set_bit(e, 16);
	} else {
1029
		/* (phased-out) F5 0x100000001 */
1030
		BN_set_bit(e, 0);
1031
		BN_set_bit(e, 32);
1032 1033
	}

1034
	if (callback == NULL) {
1035
		BN_GENCB_set_old(cb, NULL, NULL);
1036
	} else {
1037
		u.fptr = callback;
1038
		BN_GENCB_set(cb, &progress_cb, u.dptr);
1039
	}
1040

1041
	if (RSA_generate_key_ex(rsa, key->key_size, e, cb)) {
1042
		BN_free(e);
1043
		BN_GENCB_free(cb);
1044
		SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
1045 1046
#if USE_EVP
		key->keydata.pkey = pkey;
Automatic Updater's avatar
Automatic Updater committed
1047

Francis Dupont's avatar
Francis Dupont committed
1048 1049
		RSA_free(rsa);
#else
1050
		key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
1051
#endif
1052
		return (ISC_R_SUCCESS);
1053
	}
1054
	BN_GENCB_free(cb);
1055 1056
	ret = dst__openssl_toresult2("RSA_generate_key_ex",
				     DST_R_OPENSSLFAILURE);
1057

1058
 err:
Francis Dupont's avatar
Francis Dupont committed
1059 1060 1061 1062
#if USE_EVP
	if (pkey != NULL)
		EVP_PKEY_free(pkey);
#endif
1063
	if (e != NULL)
1064
		BN_free(e);
1065
	if (rsa != NULL)
1066
		RSA_free(rsa);
1067 1068
	if (cb != NULL)
		BN_GENCB_free(cb);
1069
	return (dst__openssl_toresult(ret));
1070
#else
1071 1072
	RSA *rsa;
	unsigned long e;
Francis Dupont's avatar
Francis Dupont committed
1073 1074
#if USE_EVP
	EVP_PKEY *pkey = EVP_PKEY_new();
Automatic Updater's avatar
Automatic Updater committed
1075

1076 1077
	UNUSED(callback);

Francis Dupont's avatar
Francis Dupont committed
1078 1079
	if (pkey == NULL)
		return (ISC_R_NOMEMORY);
1080 1081
#else
	UNUSED(callback);
Francis Dupont's avatar
Francis Dupont committed
1082
#endif
1083 1084

	if (exp == 0)
1085
	       e = RSA_F4;
1086
	else
1087
	       e = 0x40000003;
1088
	rsa = RSA_generate_key(key->key_size, e, NULL, NULL);
Francis Dupont's avatar
Francis Dupont committed
1089 1090 1091 1092
	if (rsa == NULL) {
#if USE_EVP
		EVP_PKEY_free(pkey);
#endif
1093 1094
		return (dst__openssl_toresult2("RSA_generate_key",
					       DST_R_OPENSSLFAILURE));
Francis Dupont's avatar
Francis Dupont committed
1095
	}
1096
	SET_FLAGS(rsa);
Francis Dupont's avatar
Francis Dupont committed
1097 1098 1099 1100 1101 1102 1103 1104 1105
#if USE_EVP
	if (!EVP_PKEY_set1_RSA(pkey, rsa)) {
		EVP_PKEY_free(pkey);
		RSA_free(rsa);
		return (dst__openssl_toresult(DST_R_OPENSSLFAILURE));
	}
	key->keydata.pkey = pkey;
	RSA_free(rsa);
#else
1106
	key->keydata.rsa = rsa;
Francis Dupont's avatar
Francis Dupont committed
1107
#endif
1108 1109

	return (ISC_R_SUCCESS);
1110
#endif
1111 1112 1113 1114
}

static isc_boolean_t
opensslrsa_isprivate(const dst_key_t *key) {
1115
	const BIGNUM *d = NULL;
Francis Dupont's avatar
Francis Dupont committed
1116 1117 1118 1119 1120 1121 1122 1123
#if USE_EVP
	RSA *rsa = EVP_PKEY_get1_RSA(key->keydata.pkey);
	INSIST(rsa != NULL);
	RSA_free(rsa);
	/* key->keydata.pkey still has a reference so rsa is still valid. */
#else
	RSA *rsa = key->keydata.rsa;
#endif
1124
	if (rsa != NULL && RSA_test_flags(rsa, RSA_FLAG_EXT_PKEY) != 0)
Francis Dupont's avatar
Francis Dupont committed
1125
		return (ISC_TRUE);
1126 1127
	RSA_get0_key(rsa, NULL, NULL, &d);
	return (ISC_TF(rsa != NULL && d != NULL));
1128 1129 1130 1131
}

static void
opensslrsa_destroy(dst_key_t *key) {
Francis Dupont's avatar
Francis Dupont committed
1132 1133