1916. [func] Integrate contibuted IDN code from JPNIC. [RT #15383]

bin/dig/dig.docbook

@@ -18,7 +18,7 @@
  - PERFORMANCE OF THIS SOFTWARE.
 -->
 
-<!-- $Id: dig.docbook,v 1.26 2005/08/30 00:45:08 marka Exp $ -->
+<!-- $Id: dig.docbook,v 1.27 2005/09/09 06:13:58 marka Exp $ -->
 <refentry id="man.dig">
 
   <refentryinfo>
@@ -874,6 +874,21 @@ dig +qr www.isc.org any -x 127.0.0.1 isc.org ns +noqr
 
   </refsect1>
 
+  <refsect1>
+    <title>IDN SUPPORT</title>
+    <para>
+      If <command>dig</command> has been built with IDN (internationalized
+      domain name) support, it can accept and display non-ASCII domain names.
+      <command>dig</command> appropriately converts character encoding of
+      domain name before sending a request to DNS server or displaying a
+      reply from the server.
+      If you'd like to turn off the IDN support for some reason, defines
+      the <envar>IDN_DISABLE</envar> environment variable.
+      The IDN support is disabled if the variable is set when 
+      <command>dig</command> runs.
+    </para>
+  </refsect1>
+
   <refsect1>
     <title>FILES</title>
     <para><filename>/etc/resolv.conf</filename>

bin/dig/dighost.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: dighost.c,v 1.285 2005/09/08 23:59:45 marka Exp $ */
+/* $Id: dighost.c,v 1.286 2005/09/09 06:13:59 marka Exp $ */
 
 /*! \file
  *  \note
@@ -33,6 +33,17 @@
 #include <string.h>
 #include <limits.h>
 
+#ifdef HAVE_LOCALE_H
+#include <locale.h>
+#endif
+
+#ifdef WITH_IDN
+#include <idn/result.h>
+#include <idn/log.h>
+#include <idn/resconf.h>
+#include <idn/api.h>
+#endif
+
 #include <dns/byaddr.h>
 #ifdef DIG_SIGCHASE
 #include <dns/dnssec.h>
@@ -123,6 +134,18 @@ int ndots = -1;
 int tries = 3;
 int lookup_counter = 0;
 
+#ifdef WITH_IDN
+static void		initialize_idn(void);
+static isc_result_t	output_filter(isc_buffer_t *buffer,
+				      unsigned int used_org,
+				      isc_boolean_t absolute);
+static idn_result_t	append_textname(char *name, const char *origin,
+					size_t namesize);
+static void		idn_check_result(idn_result_t r, const char *msg);
+
+#define MAXDLEN		256
+#endif
+
 /*%
  * Exit Codes:
  *
@@ -992,6 +1015,10 @@ setup_system(void) {
 	if (ISC_LIST_EMPTY(server_list))
 		copy_server_list(lwconf, &server_list);
 
+#ifdef WITH_IDN
+	initialize_idn();
+#endif
+
 	if (keyfile[0] != 0)
 		setup_file_key();
 	else if (keysecret[0] != 0)
@@ -1650,6 +1677,15 @@ setup_lookup(dig_lookup_t *lookup) {
 	isc_buffer_t b;
 	dns_compress_t cctx;
 	char store[MXNAME];
+#ifdef WITH_IDN
+	idn_result_t mr;
+	char utf8_textname[MXNAME], utf8_origin[MXNAME], idn_textname[MXNAME];
+#endif
+
+#ifdef WITH_IDN
+	result = dns_name_settotextfilter(output_filter);
+	check_result(result, "dns_name_settotextfilter");
+#endif
 
 	REQUIRE(lookup != NULL);
 	INSIST(!free_now);
@@ -1678,6 +1714,17 @@ setup_lookup(dig_lookup_t *lookup) {
 	isc_buffer_init(&lookup->onamebuf, lookup->onamespace,
 			sizeof(lookup->onamespace));
 
+#ifdef WITH_IDN
+	/*
+	 * We cannot convert `textname' and `origin' separately.
+	 * `textname' doesn't contain TLD, but local mapping needs
+	 * TLD.
+	 */
+	mr = idn_encodename(IDN_LOCALCONV | IDN_DELIMMAP, lookup->textname,
+			    utf8_textname, sizeof(utf8_textname));
+	idn_check_result(mr, "convert textname to UTF-8");
+#endif
+
 	/*
 	 * If the name has too many dots, force the origin to be NULL
 	 * (which produces an absolute lookup).  Otherwise, take the origin
@@ -1686,11 +1733,33 @@ setup_lookup(dig_lookup_t *lookup) {
 	 * is TRUE or we got a domain line in the resolv.conf file.
 	 */
 	/* XXX New search here? */
+#ifdef WITH_IDN
+	if ((count_dots(utf8_textname) >= ndots) || !usesearch)
+		lookup->origin = NULL; /* Force abs lookup */
+	else if (lookup->origin == NULL && lookup->new_search && usesearch)
+               lookup->origin = ISC_LIST_HEAD(search_list);
+#else
 	if ((count_dots(lookup->textname) >= ndots) || !usesearch)
 		lookup->origin = NULL; /* Force abs lookup */
 	else if (lookup->origin == NULL && lookup->new_search && usesearch)
 		lookup->origin = ISC_LIST_HEAD(search_list);
+#endif
 
+#ifdef WITH_IDN
+	if (lookup->origin != NULL) {
+		mr = idn_encodename(IDN_LOCALCONV | IDN_DELIMMAP,
+				    lookup->origin->origin, utf8_origin,
+				    sizeof(utf8_origin));
+		idn_check_result(mr, "convert origin to UTF-8");
+		mr = append_textname(utf8_textname, utf8_origin,
+				     sizeof(utf8_textname));
+		idn_check_result(mr, "append origin to textname");
+	}
+	mr = idn_encodename(IDN_LOCALMAP | IDN_NAMEPREP | IDN_ASCCHECK |
+			    IDN_IDNCONV | IDN_LENCHECK, utf8_textname,
+			    idn_textname, sizeof(idn_textname));
+	idn_check_result(mr, "convert UTF-8 textname to IDN encoding");
+#else
 	if (lookup->origin != NULL) {
 		debug("trying origin %s", lookup->origin->origin);
 		result = dns_message_gettempname(lookup->sendmsg,
@@ -1731,11 +1800,22 @@ setup_lookup(dig_lookup_t *lookup) {
 			      lookup->textname, isc_result_totext(result));
 		}
 		dns_message_puttempname(lookup->sendmsg, &lookup->oname);
-	} else {
+	} else
+#endif
+	{
 		debug("using root origin");
 		if (lookup->trace && lookup->trace_root)
 			dns_name_clone(dns_rootname, lookup->name);
 		else {
+#ifdef WITH_IDN
+			len = strlen(idn_textname);
+			isc_buffer_init(&b, idn_textname, len);
+			isc_buffer_add(&b, len);
+			result = dns_name_fromtext(lookup->name, &b,
+						   dns_rootname,
+						   ISC_FALSE,
+						   &lookup->namebuf);
+#else
 			len = strlen(lookup->textname);
 			isc_buffer_init(&b, lookup->textname, len);
 			isc_buffer_add(&b, len);
@@ -1743,6 +1823,7 @@ setup_lookup(dig_lookup_t *lookup) {
 						   dns_rootname,
 						   ISC_FALSE,
 						   &lookup->namebuf);
+#endif
 		}
 		if (result != ISC_R_SUCCESS) {
 			dns_message_puttempname(lookup->sendmsg,
@@ -3249,8 +3330,104 @@ destroy_libs(void) {
 		isc_mem_destroy(&mctx);
 }
 
- 
+#ifdef WITH_IDN
+static void
+initialize_idn(void) {
+	idn_result_t r;
+	isc_result_t result;
+
+#ifdef HAVE_SETLOCALE
+	/* Set locale */
+	(void)setlocale(LC_ALL, "");
+#endif
+	/* Create configuration context. */
+	r = idn_nameinit(1);
+	if (r != idn_success)
+		fatal("idn api initialization failed: %s",
+		      idn_result_tostring(r));
+
+	/* Set domain name -> text post-conversion filter. */
+	result = dns_name_settotextfilter(output_filter);
+	check_result(result, "dns_name_settotextfilter");
+}
+
+static isc_result_t
+output_filter(isc_buffer_t *buffer, unsigned int used_org,
+	      isc_boolean_t absolute)
+{
+	char tmp1[MAXDLEN], tmp2[MAXDLEN];
+	size_t fromlen, tolen;
+	isc_boolean_t end_with_dot;
+
+	/*
+	 * Copy contents of 'buffer' to 'tmp1', supply trailing dot
+	 * if 'absolute' is true, and terminate with NUL.
+	 */
+	fromlen = isc_buffer_usedlength(buffer) - used_org;
+	if (fromlen >= MAXDLEN)
+		return (ISC_R_SUCCESS);
+	memcpy(tmp1, (char *)isc_buffer_base(buffer) + used_org, fromlen);
+	end_with_dot = (tmp1[fromlen - 1] == '.') ? ISC_TRUE : ISC_FALSE;
+	if (absolute && !end_with_dot) {
+		fromlen++;
+		if (fromlen >= MAXDLEN)
+			return (ISC_R_SUCCESS);
+		tmp1[fromlen - 1] = '.';
+	}
+	tmp1[fromlen] = '\0';
 
+	/*
+	 * Convert contents of 'tmp1' to local encoding.
+	 */
+	if (idn_decodename(IDN_DECODE_APP, tmp1, tmp2, MAXDLEN) != idn_success)
+		return (ISC_R_SUCCESS);
+	strcpy(tmp1, tmp2);
+
+	/*
+	 * Copy the converted contents in 'tmp1' back to 'buffer'.
+	 * If we have appended trailing dot, remove it.
+	 */
+	tolen = strlen(tmp1);
+	if (absolute && !end_with_dot && tmp1[tolen - 1] == '.')
+		tolen--;
+
+	if (isc_buffer_length(buffer) < used_org + tolen)
+		return (ISC_R_NOSPACE);
+
+	isc_buffer_subtract(buffer, isc_buffer_usedlength(buffer) - used_org);
+	memcpy(isc_buffer_used(buffer), tmp1, tolen);
+	isc_buffer_add(buffer, tolen);
+
+	return (ISC_R_SUCCESS);
+}
+
+static idn_result_t
+append_textname(char *name, const char *origin, size_t namesize) {
+	size_t namelen = strlen(name);
+	size_t originlen = strlen(origin);
+
+	/* Already absolute? */
+	if (namelen > 0 && name[namelen - 1] == '.')
+		return idn_success;
+
+	/* Append dot and origin */
+
+	if (namelen + 1 + originlen >= namesize)
+		return idn_buffer_overflow;
+
+	name[namelen++] = '.';
+	(void)strcpy(name + namelen, origin);
+	return idn_success;
+}
+ 
+static void
+idn_check_result(idn_result_t r, const char *msg) {
+	if (r != idn_success) {
+		exitcode = 1;
+		fatal("%s: %s", msg, idn_result_tostring(r));
+	}
+}
+#endif /* WITH_IDN */
 
 #ifdef DIG_SIGCHASE
 void

bin/dig/host.docbook

@@ -18,7 +18,7 @@
  - PERFORMANCE OF THIS SOFTWARE.
 -->
 
-<!-- $Id: host.docbook,v 1.11 2005/08/25 00:31:32 marka Exp $ -->
+<!-- $Id: host.docbook,v 1.12 2005/09/09 06:13:59 marka Exp $ -->
 <refentry id="man.host">
 
   <refentryinfo>
@@ -237,6 +237,21 @@
     </para>
   </refsect1>
 
+  <refsect1>
+    <title>IDN SUPPORT</title>
+    <para>
+      If <command>host</command> has been built with IDN (internationalized
+      domain name) support, it can accept and display non-ASCII domain names. 
+      <command>host</command> appropriately converts character encoding of
+      domain name before sending a request to DNS server or displaying a
+      reply from the server.
+      If you'd like to turn off the IDN support for some reason, defines
+      the <envar>IDN_DISABLE</envar> environment variable.
+      The IDN support is disabled if the variable is set when
+      <command>host</command> runs.
+    </para>
+  </refsect1>
+
   <refsect1>
     <title>FILES</title>
     <para><filename>/etc/resolv.conf</filename>

config.h.in

@@ -16,7 +16,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: config.h.in,v 1.71 2005/09/09 06:13:57 marka Exp $ */
+/* $Id: config.h.in,v 1.75 2005/10/20 23:50:53 marka Exp $ */
 
 /*! \file */
 

configure.in

@@ -18,7 +18,7 @@ AC_DIVERT_PUSH(1)dnl
 esyscmd([sed "s/^/# /" COPYRIGHT])dnl
 AC_DIVERT_POP()dnl
 
-AC_REVISION($Revision: 1.392 $)
+AC_REVISION($Revision: 1.393 $)
 
 AC_INIT(lib/dns/name.c)
 AC_PREREQ(2.59)
@@ -2133,6 +2133,82 @@ then
 fi
 AC_SUBST(XSLT_DB2LATEX_ADMONITIONS)
 
+#
+# IDN support
+#
+AC_ARG_WITH(idn,
+	[  --with-idn[=MPREFIX]   enable IDN support using idnkit [default PREFIX]],
+	use_idn="$withval", use_idn="no")
+case "$use_idn" in
+yes)
+	if test X$prefix = XNONE ; then
+		idn_path=/usr/local
+	else
+		idn_path=$prefix
+	fi
+	;;
+no)
+	;;
+*)
+	idn_path="$use_idn"
+	;;
+esac
+
+iconvinc=
+iconvlib=
+AC_ARG_WITH(libiconv,
+	[  --with-libiconv[=IPREFIX]   GNU libiconv are in IPREFIX [default PREFIX]],
+	use_libiconv="$withval", use_libiconv="no")
+case "$use_libiconv" in
+yes)
+	if test X$prefix = XNONE ; then
+		iconvlib="-L/usr/local/lib -R/usr/local/lib -liconv"
+	else
+		iconvlib="-L$prefix/lib -R$prefix/lib -liconv"
+	fi
+	;;
+no)
+	iconvlib=
+	;;
+*)
+	iconvlib="-L$use_libiconv/lib -R$use_libiconv/lib -liconv"
+	;;
+esac
+
+AC_ARG_WITH(iconv,
+	[  --with-iconv[=LIBSPEC]   specify iconv library [default -liconv]],
+	iconvlib="$withval")
+case "$iconvlib" in
+no)
+	iconvlib=
+	;;
+yes)
+	iconvlib=-liconv
+	;;
+esac
+
+AC_ARG_WITH(idnlib,
+	[  --with-idnlib=ARG    specify libidnkit],
+	idnlib="$withval", idnlib="no")
+if test "$idnlib" = yes; then
+	AC_MSG_ERROR([You must specify ARG for --with-idnlib.])
+fi
+
+IDNLIBS=
+if test "$use_idn" != no; then
+	AC_DEFINE(WITH_IDN, 1, [define if idnkit support is to be included.])
+	STD_CINCLUDES="$STD_CINCLUDES -I$idn_path/include"
+	if test "$idnlib" != no; then
+		IDNLIBS="$idnlib $iconvlib"
+	else
+		IDNLIBS="-L$idn_path/lib -lidnkit $iconvlib"
+	fi
+fi
+AC_SUBST(IDNLIBS)
+
+AC_CHECK_HEADERS(locale.h)
+AC_CHECK_FUNCS(setlocale)
+
 #
 # Substitutions
 #

lib/dns/include/dns/name.h

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: name.h,v 1.117 2005/09/05 00:11:03 marka Exp $ */
+/* $Id: name.h,v 1.118 2005/09/09 06:14:00 marka Exp $ */
 
 #ifndef DNS_NAME_H
 #define DNS_NAME_H 1
@@ -156,6 +156,15 @@ LIBDNS_EXTERNAL_DATA extern dns_name_t *dns_wildcardname;
  */
 #define DNS_NAME_MAXWIRE 255
 
+/*
+ * Text output filter procedure.
+ * 'target' is the buffer to be converted.  The region to be converted
+ * is from 'buffer'->base + 'used_org' to the end of the used region.
+ */
+typedef isc_result_t (*dns_name_totextfilter_t)(isc_buffer_t *target,
+						unsigned int used_org,
+						isc_boolean_t absolute);
+
 /***
  *** Initialization
  ***/
@@ -1117,6 +1126,17 @@ dns_name_format(dns_name_t *name, char *cp, unsigned int size);
  *
  */
 
+isc_result_t
+dns_name_settotextfilter(dns_name_totextfilter_t proc);
+/*%<
+ * Set / clear a thread specific function 'proc' to be called at the
+ * end of dns_name_totext().
+ *
+ * Returns
+ *\li	#ISC_R_SUCCESS
+ *\li	#ISC_R_UNEXPECTED
+ */
+
 #define DNS_NAME_FORMATSIZE (DNS_NAME_MAXTEXT + 1)
 /*%<
  * Suggested size of buffer passed to dns_name_format().

lib/dns/name.c

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: name.c,v 1.151 2005/07/20 01:46:49 marka Exp $ */
+/* $Id: name.c,v 1.152 2005/09/09 06:13:59 marka Exp $ */
 
 /*! \file */
 
@@ -26,8 +26,10 @@
 #include <isc/buffer.h>
 #include <isc/hash.h>
 #include <isc/mem.h>
+#include <isc/once.h>
 #include <isc/print.h>
 #include <isc/string.h>
+#include <isc/thread.h>
 #include <isc/util.h>
 
 #include <dns/compress.h>
@@ -184,6 +186,15 @@ LIBDNS_EXTERNAL_DATA dns_name_t *dns_wildcardname = &wild;
 unsigned int
 dns_fullname_hash(dns_name_t *name, isc_boolean_t case_sensitive);
 
+/*
+ * dns_name_t to text post-conversion procedure.
+ */
+#ifdef ISC_PLATFORM_USETHREADS
+static isc_thread_key_t totext_filter_proc_key;
+#else
+static dns_name_totextfilter_t totext_filter_proc = NULL;
+#endif
+
 static void
 set_offsets(const dns_name_t *name, unsigned char *offsets,
 	    dns_name_t *set_name);
@@ -1273,6 +1284,10 @@ dns_name_totext(dns_name_t *name, isc_boolean_t omit_final_dot,
 	unsigned int trem, count;
 	unsigned int labels;
 	isc_boolean_t saw_root = ISC_FALSE;
+	unsigned int oused = target->used;
+#ifdef ISC_PLATFORM_USETHREADS
+	dns_name_totextfilter_t totext_filter_proc;
+#endif
 
 	/*
 	 * This function assumes the name is in proper uncompressed
@@ -1412,6 +1427,12 @@ dns_name_totext(dns_name_t *name, isc_boolean_t omit_final_dot,
 
 	isc_buffer_add(target, tlen - trem);
 
+#ifdef ISC_PLATFORM_USETHREADS
+	totext_filter_proc = isc_key_getspecific(totext_filter_proc_key);
+#endif
+	if (totext_filter_proc != NULL)
+		return ((*totext_filter_proc)(target, oused, saw_root));
+
 	return (ISC_R_SUCCESS);
 }
 
@@ -2193,6 +2214,31 @@ dns_name_print(dns_name_t *name, FILE *stream) {
 	return (ISC_R_SUCCESS);
 }
 
+#ifdef ISC_PLATFORM_USETHREADS
+static void
+totext_filter_proc_key_init(void) {
+	RUNTIME_CHECK(isc_key_create(&totext_filter_proc_key, NULL) == 0);
+}
+#endif
+
+isc_result_t
+dns_name_settotextfilter(dns_name_totextfilter_t proc) {
+#ifdef ISC_PLATFORM_USETHREADS
+	static isc_once_t once = ISC_ONCE_INIT;
+	isc_result_t result;
+
+	result = isc_once_do(&once, totext_filter_proc_key_init);
+	if (result != ISC_R_SUCCESS)
+		return (result);
+	if (isc_key_setspecific(totext_filter_proc_key, proc) != 0)
+		result = ISC_R_UNEXPECTED;
+	return (result);
+#else
+	totext_filter_proc = proc;
+	return (ISC_R_SUCCESS);
+#endif
+}
+
 void
 dns_name_format(dns_name_t *name, char *cp, unsigned int size) {
 	isc_result_t result;

lib/dns/win32/libdns.def

@@ -329,6 +329,7 @@ dns_name_print
 dns_name_rdatacompare
 dns_name_reset
 dns_name_setbuffer
+dns_name_settotextfilter
 dns_name_split
 dns_name_tofilenametext
 dns_name_toregion

lib/isc/pthreads/include/isc/thread.h

@@ -15,7 +15,7 @@
  * PERFORMANCE OF THIS SOFTWARE.
  */
 
-/* $Id: thread.h,v 1.22 2005/04/29 00:23:49 marka Exp $ */
+/* $Id: thread.h,v 1.23 2005/09/09 06:14:00 marka Exp $ */
 
 #ifndef ISC_THREAD_H
 #define ISC_THREAD_H 1
@@ -33,6 +33,7 @@ typedef pthread_t isc_thread_t;
 typedef void * isc_threadresult_t;
 typedef void * isc_threadarg_t;
 typedef isc_threadresult_t (*isc_threadfunc_t)(isc_threadarg_t);
+typedef pthread_key_t isc_thread_key_t;
 
 isc_result_t
 isc_thread_create(isc_threadfunc_t, isc_threadarg_t, isc_thread_t *);
@@ -49,6 +50,11 @@ isc_thread_setconcurrency(unsigned int level);
 #define isc_thread_self \
 	(unsigned long)pthread_self
 
+#define isc_key_create pthread_key_create
+#define isc_key_getspecific pthread_getspecific
+#define isc_key_setspecific pthread_setspecific
+#define isc_key_delete pthread_key_delete
+
 ISC_LANG_ENDDECLS
 
 #endif /* ISC_THREAD_H */