Merged rt46864 (check MD5 amd SHA1 support)

CHANGES

+4866.	[port]		DST library initialization verifies MD5 (when MD5
+			was not disabled) and SHA-1 hash and HMAC support.
+			[RT #46764]
+
 4865.	[cleanup]	Simplify handling isc_socket_sendto2() return values.
 			[RT #46986]
 

config.h.in

@@ -120,6 +120,9 @@ int sigwait(const unsigned int *set, int *sig);
 /** define if you have strerror in the C library. */
 #undef HAVE_STRERROR
 
+/* Define if OpenSSL provides FIPS_mode() */
+#undef HAVE_FIPS_MODE
+
 /* Define if OpenSSL includes DSA support */
 #undef HAVE_OPENSSL_DSA
 

config.h.win32

@@ -318,6 +318,9 @@ typedef __int64 off_t;
 /* Define to 1 if you have the `EVP_sha512' function. */
 @HAVE_EVP_SHA512@
 
+/* Define if OpenSSL provides FIPS_mode() */
+@HAVE_FIPS_MODE@
+
 /* Define if OpenSSL includes DSA support */
 @HAVE_OPENSSL_DSA@
 

configure.in

@@ -1789,6 +1789,21 @@ no)
 ;;
 esac
 
+	AC_MSG_CHECKING(for OpenSSL FIPS mode support)
+	have_fips_mode=""
+	AC_TRY_LINK([#include <openssl/crypto.h>],
+		    [FIPS_mode();],
+		    have_fips_mode=yes,
+		    have_fips_mode=no)
+	if test "x$have_fips_mode" = "xyes"
+	then
+		AC_DEFINE([HAVE_FIPS_MODE], [1],
+			  [Define if OpenSSL provides FIPS_mode()])
+		AC_MSG_RESULT(yes)
+	else
+		AC_MSG_RESULT(no)
+	fi
+
 	AC_MSG_CHECKING(for OpenSSL DSA support)
 	if test -f $use_openssl/include/openssl/dsa.h
 	then

lib/dns/hmac_link.c

@@ -22,7 +22,6 @@
 
 /*
  * Principal Author: Brian Wellington
- * $Id: hmac_link.c,v 1.19 2011/01/11 23:47:13 tbox Exp $
  */
 
 #include <config.h>
@@ -42,6 +41,7 @@
 #include <dst/result.h>
 
 #include "dst_internal.h"
+#include "dst_openssl.h"
 #include "dst_parse.h"
 
 #ifndef PK11_MD5_DISABLE
@@ -337,6 +337,28 @@ static dst_func_t hmacmd5_functions = {
 
 isc_result_t
 dst__hmacmd5_init(dst_func_t **funcp) {
+#ifdef HAVE_FIPS_MODE
+	/*
+	 * Problems from OpenSSL are likely from FIPS mode
+	 */
+	int fips_mode = FIPS_mode();
+
+	if (fips_mode != 0) {
+		UNEXPECTED_ERROR(__FILE__, __LINE__,
+				 "FIPS mode is %d: MD5 is only supported "
+				 "if the value is 0.\n"
+				 "Please disable either FIPS mode or MD5.",
+				 fips_mode);
+	}
+#endif
+
+	/*
+	 * Prevent use of incorrect crypto
+	 */
+
+	RUNTIME_CHECK(isc_md5_check(ISC_FALSE));
+	RUNTIME_CHECK(isc_hmacmd5_check(0));
+
 	REQUIRE(funcp != NULL);
 	if (*funcp == NULL)
 		*funcp = &hmacmd5_functions;
@@ -623,6 +645,12 @@ static dst_func_t hmacsha1_functions = {
 
 isc_result_t
 dst__hmacsha1_init(dst_func_t **funcp) {
+	/*
+	 * Prevent use of incorrect crypto
+	 */
+	RUNTIME_CHECK(isc_sha1_check(ISC_FALSE));
+	RUNTIME_CHECK(isc_hmacsha1_check(0));
+
 	REQUIRE(funcp != NULL);
 	if (*funcp == NULL)
 		*funcp = &hmacsha1_functions;

lib/isc/hmacmd5.c

@@ -334,6 +334,72 @@ isc_hmacmd5_verify2(isc_hmacmd5_t *ctx, unsigned char *digest, size_t len) {
 	return (isc_safe_memequal(digest, newdigest, len));
 }
 
+/*
+ * Check for MD5 support; if it does not work, raise a fatal error.
+ *
+ * Use the first test vector from RFC 2104, with a second round using
+ * a too-short key.
+ *
+ * Standard use is testing 0 and expecting result true.
+ * Testing use is testing 1..4 and expecting result false.
+ */
+isc_boolean_t
+isc_hmacmd5_check(int testing) {
+	isc_hmacmd5_t ctx;
+	unsigned char key[] = { /* 0x0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b0b */
+		0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
+		0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b
+	};
+	unsigned char input[] = { /* "Hi There" */
+		0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65
+	};
+	unsigned char expected[] = {
+		0x92, 0x94, 0x72, 0x7a, 0x36, 0x38, 0xbb, 0x1c,
+		0x13, 0xf4, 0x8e, 0xf8, 0x15, 0x8b, 0xfc, 0x9d
+	};
+	unsigned char expected2[] = {
+		0xad, 0xb8, 0x48, 0x05, 0xb8, 0x8d, 0x03, 0xe5,
+		0x90, 0x1e, 0x4b, 0x05, 0x69, 0xce, 0x35, 0xea
+	};
+	isc_boolean_t result;
+
+	/*
+	 * Introduce a fault for testing.
+	 */
+	switch (testing) {
+	case 0:
+	default:
+		break;
+	case 1:
+		key[0] ^= 0x01;
+		break;
+	case 2:
+		input[0] ^= 0x01;
+		break;
+	case 3:
+		expected[0] ^= 0x01;
+		break;
+	case 4:
+		expected2[0] ^= 0x01;
+		break;
+	}
+
+	/*
+	 * These functions do not return anything; any failure will be fatal.
+	 */
+	isc_hmacmd5_init(&ctx, key, 16U);
+	isc_hmacmd5_update(&ctx, input, 8U);
+	result = isc_hmacmd5_verify2(&ctx, expected, sizeof(expected));
+	if (!result) {
+		return (result);
+	}
+
+	/* Second round using a byte key */
+	isc_hmacmd5_init(&ctx, key, 1U);
+	isc_hmacmd5_update(&ctx, input, 8U);
+	return (isc_hmacmd5_verify2(&ctx, expected2, sizeof(expected2)));
+}
+
 #else /* !PK11_MD5_DISABLE */
 #ifdef WIN32
 /* Make the Visual Studio linker happy */
@@ -345,5 +411,6 @@ void isc_hmacmd5_sign() { INSIST(0); }
 void isc_hmacmd5_update() { INSIST(0); }
 void isc_hmacmd5_verify() { INSIST(0); }
 void isc_hmacmd5_verify2() { INSIST(0); }
+void isc_hmacmd5_check() { INSIST(0); }
 #endif
 #endif /* PK11_MD5_DISABLE */

lib/isc/hmacsha.c

@@ -1497,3 +1497,72 @@ isc_hmacsha512_verify(isc_hmacsha512_t *ctx, unsigned char *digest, size_t len)
 	isc_hmacsha512_sign(ctx, newdigest, ISC_SHA512_DIGESTLENGTH);
 	return (isc_safe_memequal(digest, newdigest, len));
 }
+
+/*
+ * Check for SHA-1 support; if it does not work, raise a fatal error.
+ *
+ * Use the first test vector from RFC 2104, with a second round using
+ * a too-short key.
+ *
+ * Standard use is testing 0 and expecting result true.
+ * Testing use is testing 1..4 and expecting result false.
+ */
+isc_boolean_t
+isc_hmacsha1_check(int testing) {
+	isc_hmacsha1_t ctx;
+	unsigned char key[] = { /* 20*0x0b */
+		0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
+		0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b, 0x0b,
+		0x0b, 0x0b, 0x0b, 0x0b
+	};
+	unsigned char input[] = { /* "Hi There" */
+		0x48, 0x69, 0x20, 0x54, 0x68, 0x65, 0x72, 0x65
+	};
+	unsigned char expected[] = {
+		0xb6, 0x17, 0x31, 0x86, 0x55, 0x05, 0x72, 0x64,
+		0xe2, 0x8b, 0xc0, 0xb6, 0xfb, 0x37, 0x8c, 0x8e,
+		0xf1, 0x46, 0xbe, 0x00
+	};
+	unsigned char expected2[] = {
+		0xa0, 0x75, 0xe0, 0x5f, 0x7f, 0x17, 0x9d, 0x34,
+		0xb2, 0xab, 0xc5, 0x19, 0x8f, 0x38, 0x62, 0x36,
+		0x42, 0xbd, 0xec, 0xde
+	};
+	isc_boolean_t result;
+
+	/*
+	 * Introduce a fault for testing.
+	 */
+	switch (testing) {
+	case 0:
+	default:
+		break;
+	case 1:
+		key[0] ^= 0x01;
+		break;
+	case 2:
+		input[0] ^= 0x01;
+		break;
+	case 3:
+		expected[0] ^= 0x01;
+		break;
+	case 4:
+		expected2[0] ^= 0x01;
+		break;
+	}
+
+	/*
+	 * These functions do not return anything; any failure will be fatal.
+	 */
+	isc_hmacsha1_init(&ctx, key, 20U);
+	isc_hmacsha1_update(&ctx, input, 8U);
+	result = isc_hmacsha1_verify(&ctx, expected, sizeof(expected));
+	if (!result) {
+		return (result);
+	}
+
+	/* Second round using a byte key */
+	isc_hmacsha1_init(&ctx, key, 1U);
+	isc_hmacsha1_update(&ctx, input, 8U);
+	return (isc_hmacsha1_verify(&ctx, expected2, sizeof(expected2)));
+}

lib/isc/include/isc/hmacmd5.h

@@ -73,6 +73,9 @@ isc_hmacmd5_verify(isc_hmacmd5_t *ctx, unsigned char *digest);
 isc_boolean_t
 isc_hmacmd5_verify2(isc_hmacmd5_t *ctx, unsigned char *digest, size_t len);
 
+isc_boolean_t
+isc_hmacmd5_check(int testing);
+
 ISC_LANG_ENDDECLS
 
 #endif /* !PK11_MD5_DISABLE */

lib/isc/include/isc/hmacsha.h

@@ -101,6 +101,9 @@ isc_hmacsha1_sign(isc_hmacsha1_t *ctx, unsigned char *digest, size_t len);
 isc_boolean_t
 isc_hmacsha1_verify(isc_hmacsha1_t *ctx, unsigned char *digest, size_t len);
 
+isc_boolean_t
+isc_hmacsha1_check(int testing);
+
 
 void
 isc_hmacsha224_init(isc_hmacsha224_t *ctx, const unsigned char *key,

lib/isc/include/isc/md5.h

@@ -84,6 +84,9 @@ isc_md5_update(isc_md5_t *ctx, const unsigned char *buf, unsigned int len);
 void
 isc_md5_final(isc_md5_t *ctx, unsigned char *digest);
 
+isc_boolean_t
+isc_md5_check(isc_boolean_t testing);
+
 ISC_LANG_ENDDECLS
 
 #endif /* !PK11_MD5_DISABLE */

lib/isc/include/isc/sha1.h

@@ -65,6 +65,9 @@ isc_sha1_update(isc_sha1_t *ctx, const unsigned char *data, unsigned int len);
 void
 isc_sha1_final(isc_sha1_t *ctx, unsigned char *digest);
 
+isc_boolean_t
+isc_sha1_check(isc_boolean_t testing);
+
 ISC_LANG_ENDDECLS
 
 #endif /* ISC_SHA1_H */

lib/isc/md5.c

@@ -55,7 +55,9 @@ void
 isc_md5_init(isc_md5_t *ctx) {
 	ctx->ctx = EVP_MD_CTX_new();
 	RUNTIME_CHECK(ctx->ctx != NULL);
-	RUNTIME_CHECK(EVP_DigestInit(ctx->ctx, EVP_md5()) == 1);
+	if (EVP_DigestInit(ctx->ctx, EVP_md5()) != 1) {
+		FATAL_ERROR(__FILE__, __LINE__, "Cannot initialize MD5.");
+	}
 }
 
 void
@@ -335,6 +337,46 @@ isc_md5_final(isc_md5_t *ctx, unsigned char *digest) {
 }
 #endif
 
+/*
+ * Check for MD5 support; if it does not work, raise a fatal error.
+ *
+ * Use "a" as the test vector.
+ *
+ * Standard use is testing false and result true.
+ * Testing use is testing true and result false;
+ */
+isc_boolean_t
+isc_md5_check(isc_boolean_t testing) {
+	isc_md5_t ctx;
+	unsigned char input = 'a';
+	unsigned char digest[ISC_MD5_DIGESTLENGTH];
+	unsigned char expected[] = {
+		0x0c, 0xc1, 0x75, 0xb9, 0xc0, 0xf1, 0xb6, 0xa8,
+		0x31, 0xc3, 0x99, 0xe2, 0x69, 0x77, 0x26, 0x61
+	};
+
+	INSIST(sizeof(expected) == ISC_MD5_DIGESTLENGTH);
+
+	/*
+	 * Introduce a fault for testing.
+	 */
+	if (testing) {
+		input ^= 0x01;
+	}
+
+	/*
+	 * These functions do not return anything; any failure will be fatal.
+	 */
+	isc_md5_init(&ctx);
+	isc_md5_update(&ctx, &input, 1U);
+	isc_md5_final(&ctx, digest);
+
+	/*
+	 * Must return true in standard case, should return false for testing.
+	 */
+	return (ISC_TF(bcmp(digest, expected, ISC_MD5_DIGESTLENGTH) == 0));
+}
+
 #else /* !PK11_MD5_DISABLE */
 #ifdef WIN32
 /* Make the Visual Studio linker happy */
@@ -344,5 +386,6 @@ void isc_md5_final() { INSIST(0); }
 void isc_md5_init() { INSIST(0); }
 void isc_md5_invalidate() { INSIST(0); }
 void isc_md5_update() { INSIST(0); }
+void isc_md5_check() { INSIST(0); }
 #endif
 #endif /* PK11_MD5_DISABLE */

lib/isc/sha1.c

@@ -52,7 +52,9 @@ isc_sha1_init(isc_sha1_t *context)
 
 	context->ctx = EVP_MD_CTX_new();
 	RUNTIME_CHECK(context->ctx != NULL);
-	RUNTIME_CHECK(EVP_DigestInit(context->ctx, EVP_sha1()) == 1);
+	if (EVP_DigestInit(context->ctx, EVP_sha1()) != 1) {
+		FATAL_ERROR(__FILE__, __LINE__, "Cannot initialize SHA1.");
+	}
 }
 
 void
@@ -404,3 +406,44 @@ isc_sha1_final(isc_sha1_t *context, unsigned char *digest) {
 	isc_safe_memwipe(context, sizeof(*context));
 }
 #endif
+
+/*
+ * Check for SHA-1 support; if it does not work, raise a fatal error.
+ *
+ * Use "a" as the test vector.
+ *
+ * Standard use is testing false and result true.
+ * Testing use is testing true and result false;
+ */
+isc_boolean_t
+isc_sha1_check(isc_boolean_t testing) {
+	isc_sha1_t ctx;
+	unsigned char input = 'a';
+	unsigned char digest[ISC_SHA1_DIGESTLENGTH];
+	unsigned char expected[] = {
+		0x86, 0xf7, 0xe4, 0x37, 0xfa, 0xa5, 0xa7, 0xfc,
+		0xe1, 0x5d, 0x1d, 0xdc, 0xb9, 0xea, 0xea, 0xea,
+		0x37, 0x76, 0x67, 0xb8
+	};
+
+	INSIST(sizeof(expected) == ISC_SHA1_DIGESTLENGTH);
+
+	/*
+	 * Introduce a fault for testing.
+	 */
+	if (testing) {
+		input ^= 0x01;
+	}
+
+	/*
+	 * These functions do not return anything; any failure will be fatal.
+	 */
+	isc_sha1_init(&ctx);
+	isc_sha1_update(&ctx, &input, 1U);
+	isc_sha1_final(&ctx, digest);
+
+	/*
+	 * Must return true in standard case, should return false for testing.
+	 */
+	return (ISC_TF(bcmp(digest, expected, ISC_SHA1_DIGESTLENGTH) == 0));
+}

lib/isc/sha2.c

@@ -73,7 +73,9 @@ isc_sha224_init(isc_sha224_t *context) {
 	}
 	context->ctx = EVP_MD_CTX_new();
 	RUNTIME_CHECK(context->ctx != NULL);
-	RUNTIME_CHECK(EVP_DigestInit(context->ctx, EVP_sha224()) == 1);
+	if (EVP_DigestInit(context->ctx, EVP_sha224()) != 1) {
+		FATAL_ERROR(__FILE__, __LINE__, "Cannot initialize SHA224.");
+	}
 }
 
 void
@@ -119,7 +121,9 @@ isc_sha256_init(isc_sha256_t *context) {
 	}
 	context->ctx = EVP_MD_CTX_new();
 	RUNTIME_CHECK(context->ctx != NULL);
-	RUNTIME_CHECK(EVP_DigestInit(context->ctx, EVP_sha256()) == 1);
+	if (EVP_DigestInit(context->ctx, EVP_sha256()) != 1) {
+		FATAL_ERROR(__FILE__, __LINE__, "Cannot initialize SHA256.");
+	}
 }
 
 void
@@ -165,7 +169,9 @@ isc_sha512_init(isc_sha512_t *context) {
 	}
 	context->ctx = EVP_MD_CTX_new();
 	RUNTIME_CHECK(context->ctx != NULL);
-	RUNTIME_CHECK(EVP_DigestInit(context->ctx, EVP_sha512()) == 1);
+	if (EVP_DigestInit(context->ctx, EVP_sha512()) != 1) {
+		FATAL_ERROR(__FILE__, __LINE__, "Cannot initialize SHA512.");
+	}
 }
 
 void
@@ -209,7 +215,9 @@ isc_sha384_init(isc_sha384_t *context) {
 	}
 	context->ctx = EVP_MD_CTX_new();
 	RUNTIME_CHECK(context->ctx != NULL);
-	RUNTIME_CHECK(EVP_DigestInit(context->ctx, EVP_sha384()) == 1);
+	if (EVP_DigestInit(context->ctx, EVP_sha384()) != 1) {
+		FATAL_ERROR(__FILE__, __LINE__, "Cannot initialize SHA384.");
+	}
 }
 
 void

lib/isc/tests/hash_test.c

@@ -1960,6 +1960,42 @@ ATF_TC_BODY(isc_hash_initializer, tc) {
 	ATF_CHECK_EQ(h1, h2);
 }
 
+#ifndef PK11_MD5_DISABLE
+ATF_TC(md5_check);
+ATF_TC_HEAD(md5_check, tc) {
+	atf_tc_set_md_var(tc, "descr", "Startup MD5 check test");
+}
+ATF_TC_BODY(md5_check, tc) {
+	UNUSED(tc);
+
+	ATF_REQUIRE(isc_md5_check(ISC_FALSE));
+	ATF_CHECK(!isc_md5_check(ISC_TRUE));
+
+	ATF_REQUIRE(isc_hmacmd5_check(0));
+	ATF_CHECK(!isc_hmacmd5_check(1));
+	ATF_CHECK(!isc_hmacmd5_check(2));
+	ATF_CHECK(!isc_hmacmd5_check(3));
+	ATF_CHECK(!isc_hmacmd5_check(4));
+}
+#endif
+
+ATF_TC(sha1_check);
+ATF_TC_HEAD(sha1_check, tc) {
+	atf_tc_set_md_var(tc, "descr", "Startup SHA-1 check test");
+}
+ATF_TC_BODY(sha1_check, tc) {
+	UNUSED(tc);
+
+	ATF_REQUIRE(isc_sha1_check(ISC_FALSE));
+	ATF_CHECK(!isc_sha1_check(ISC_TRUE));
+
+	ATF_REQUIRE(isc_hmacsha1_check(0));
+	ATF_CHECK(!isc_hmacsha1_check(1));
+	ATF_CHECK(!isc_hmacsha1_check(2));
+	ATF_CHECK(!isc_hmacsha1_check(3));
+	ATF_CHECK(!isc_hmacsha1_check(4));
+}
+
 /*
  * Main
  */
@@ -1968,6 +2004,11 @@ ATF_TP_ADD_TCS(tp) {
 	 * Tests of hash functions, including isc_hash and the
 	 * various cryptographic hashes.
 	 */
+#ifndef PK11_MD5_DISABLE
+	ATF_TP_ADD_TC(tp, md5_check);
+#endif
+	ATF_TP_ADD_TC(tp, sha1_check);
+
 	ATF_TP_ADD_TC(tp, isc_hash_function);
 	ATF_TP_ADD_TC(tp, isc_hash_function_reverse);
 	ATF_TP_ADD_TC(tp, isc_hash_initializer);

lib/isc/win32/libisc.def.in

@@ -286,12 +286,14 @@ isc_heap_insert
 isc_hex_decodestring
 isc_hex_tobuffer
 isc_hex_totext
+isc_hmacmd5_check
 isc_hmacmd5_init
 isc_hmacmd5_invalidate
 isc_hmacmd5_sign
 isc_hmacmd5_update
 isc_hmacmd5_verify
 isc_hmacmd5_verify2
+isc_hmacsha1_check
 isc_hmacsha1_init
 isc_hmacsha1_invalidate
 isc_hmacsha1_sign
@@ -407,6 +409,7 @@ isc_logconfig_destroy
 isc_logconfig_get
 isc_logconfig_use
 isc_logfile_roll
+isc_md5_check
 isc_md5_final
 isc_md5_init
 isc_md5_invalidate
@@ -580,6 +583,7 @@ isc_serial_gt
 isc_serial_le
 isc_serial_lt
 isc_serial_ne
+isc_sha1_check
 isc_sha1_final
 isc_sha1_init
 isc_sha1_invalidate

win32utils/Configure

@@ -224,6 +224,7 @@ my @substdefh = ("AES_CC",
                  "HAVE_EVP_SHA256",
                  "HAVE_EVP_SHA384",
                  "HAVE_EVP_SHA512",
+                 "HAVE_FIPS_MODE",
                  "HAVE_GEOIP",
                  "HAVE_GEOIP_CITY_V6",
                  "HAVE_GEOIP_V6",
@@ -1613,6 +1614,33 @@ EOF
     }
 }
 
+# check FIPS_mode
+
+if ($use_openssl eq "yes") {
+    if ($verbose) {
+        printf "checking for FIPS_mode\n";
+    }
+    open F, ">testfips.c" || die $!;
+    print F << 'EOF';
+extern int FIPS_mode();
+
+int main() {
+    return FIPS_mode();
+}
+EOF
+    close F;
+    my $library = $configlib{"OPENSSL_LIB"};
+    $compret = `cl /nologo /MD testfips.c "$library"`;
+    if (grep { -f and -x } ".\\testfips.exe") {
+        $configdefh{"HAVE_FIPS_MODE"} = 1;
+    } else {
+        if ($verbose) {
+            print "can't compile FIPS_mode test: $compret\n";
+            print "disabling FIPS_mode\n";
+        }
+    }
+}
+
 # check EVP_sha256 / EVP_sha384 / EVP_sha512
 
 if ($use_openssl eq "yes") {
@@ -3428,7 +3456,7 @@ exit 0;
 #  --with-dlz-* ?
 #
 # Notes: MSVC versions
-# MSVC 15.0 _MSC_VER == 1910 (VS 2017)
+# MSVC 15.x _MSC_VER == 191y (VS 2017)
 # MSVC 14.0 _MSC_VER == 1900 (VS 2015)
 # MSVC 12.0 _MSC_VER == 1800 (VS 2013)
 # MSVC 11.0 _MSC_VER == 1700 (VS 2012)