make maxbits signed as BN_num_bits is signed

lib/dns/dst_internal.h

@@ -170,7 +170,7 @@ struct dst_func {
 	 */
 	isc_result_t (*sign)(dst_context_t *dctx, isc_buffer_t *sig);
 	isc_result_t (*verify)(dst_context_t *dctx, const isc_region_t *sig);
-	isc_result_t (*verify2)(dst_context_t *dctx, unsigned int maxbits,
+	isc_result_t (*verify2)(dst_context_t *dctx, int maxbits,
 				const isc_region_t *sig);
 	isc_result_t (*computesecret)(const dst_key_t *pub,
 				      const dst_key_t *priv,

lib/dns/opensslrsa_link.c

@@ -511,7 +511,7 @@ opensslrsa_sign(dst_context_t *dctx, isc_buffer_t *sig) {
 }
 
 static isc_result_t
-opensslrsa_verify2(dst_context_t *dctx, unsigned int maxbits, const isc_region_t *sig) {
+opensslrsa_verify2(dst_context_t *dctx, int maxbits, const isc_region_t *sig) {
 	dst_key_t *key = dctx->key;
 	int status = 0;
 #if USE_EVP