Commit eb48e662 authored by Ondřej Surý's avatar Ondřej Surý
Browse files

Merge branch '26-switch-to-IDNA2008-non-transitional-processing-v9_11' into 'v9_11'

Resolve "Switch to IDNA2008 non-transitional processing (and use libidn2 for that)"

Closes #26

See merge request !124
parents f0cfa8ac 2ff3b664
Pipeline #810 passed with stages
in 6 minutes and 22 seconds
......@@ -88,7 +88,7 @@ stages:
- test -w "${CCACHE_DIR}" && export PATH="/usr/lib/ccache:${PATH}"
- autoreconf -fi
script:
- ./configure --enable-developer --with-libtool --disable-static --with-atf=/usr/local
- ./configure --enable-developer --with-libtool --disable-static --with-atf=/usr/local --with-libidn2
- make -j${PARALLEL_JOBS_BUILD:-1} -k all V=1
artifacts:
expire_in: '1 hour'
......
4915. [func] Implement IDNA2008 support in dig by adding support
for libidn2. New dig option +idnin has been added,
which allows to process invalid domain names much
like dig without IDN support. libidn2 version 2.0
or higher is needed for +idnout enabled by default.
4913. [test] Re-implemented older unit tests in bin/tests as ATF,
removed the lib/tests unit testing library. [GL #115]
......
......@@ -19,7 +19,7 @@ READLINE_LIB = @READLINE_LIB@
CINCLUDES = -I${srcdir}/include ${DNS_INCLUDES} \
${BIND9_INCLUDES} ${ISC_INCLUDES} \
${LWRES_INCLUDES} ${ISCCFG_INCLUDES} @DST_OPENSSL_INC@
${LWRES_INCLUDES} ${ISCCFG_INCLUDES} @LIBIDN2_CFLAGS@ @DST_OPENSSL_INC@
CDEFINES = -DVERSION=\"${VERSION}\" @CRYPTO@
CWARNINGS =
......@@ -41,10 +41,10 @@ DEPLIBS = ${DNSDEPLIBS} ${BIND9DEPLIBS} ${ISCDEPLIBS} \
${ISCCFGDEPLIBS} ${LWRESDEPLIBS}
LIBS = ${LWRESLIBS} ${BIND9LIBS} ${ISCCFGLIBS} \
${ISCLIBS} @IDNLIBS@ @LIBS@
${ISCLIBS} @IDNKIT_LIBS@ @LIBIDN2_LIBS@ @LIBS@
NOSYMLIBS = ${LWRESLIBS} ${BIND9LIBS} ${ISCCFGLIBS} \
${ISCNOSYMLIBS} @IDNLIBS@ @LIBS@
${ISCNOSYMLIBS} @IDNKIT_LIBS@ @LIBIDN2_LIBS@ @LIBS@
SUBDIRS =
......
......@@ -190,7 +190,8 @@ help(void) {
" +[no]fail (Don't try next server on SERVFAIL)\n"
" +[no]header-only (Send query without a question section)\n"
" +[no]identify (ID responders in short answers)\n"
" +[no]idnout (convert IDN response)\n"
" +[no]idnin (Parse IDN names)\n"
" +[no]idnout (Convert IDN response)\n"
" +[no]ignore (Don't revert to TCP for TC responses.)\n"
" +[no]keepopen (Keep the TCP socket open between queries)\n"
" +[no]mapped (Allow mapped IPv4 over IPv6)\n"
......@@ -1094,12 +1095,28 @@ plus_option(const char *option, isc_boolean_t is_batchfile,
lookup->identify = state;
break;
case 'n':
FULLCHECK("idnout");
#ifndef WITH_IDN
fprintf(stderr, ";; IDN support not enabled\n");
switch (cmd[3]) {
case 'i':
FULLCHECK("idnin");
#ifndef WITH_IDN_SUPPORT
fprintf(stderr, ";; IDN input support"
" not enabled\n");
#else
lookup->idnin = state;
#endif
break;
case 'o':
FULLCHECK("idnout");
#ifndef WITH_IDN_OUT_SUPPORT
fprintf(stderr, ";; IDN output support"
" not enabled\n");
#else
lookup->idnout = state;
lookup->idnout = state;
#endif
break;
default:
goto invalid_option;
}
break;
default:
goto invalid_option;
......
......@@ -776,6 +776,17 @@
</listitem>
</varlistentry>
<varlistentry>
<term><option>+[no]idnin</option></term>
<listitem>
<para>
Process [do not process] IDN domain names on input.
This requires IDN SUPPORT to have been enabled at
compile time. The default is to process IDN input.
</para>
</listitem>
</varlistentry>
<varlistentry>
<term><option>+[no]idnout</option></term>
<listitem>
......@@ -1268,10 +1279,9 @@ dig +qr www.isc.org any -x 127.0.0.1 isc.org ns +noqr
<command>dig</command> appropriately converts character encoding of
domain name before sending a request to DNS server or displaying a
reply from the server.
If you'd like to turn off the IDN support for some reason, defines
the <envar>IDN_DISABLE</envar> environment variable.
The IDN support is disabled if the variable is set when
<command>dig</command> runs.
If you'd like to turn off the IDN support for some reason, use
parameters <parameter>+noidnin</parameter> and
<parameter>+noidnout</parameter>.
</para>
</refsection>
......
......@@ -24,18 +24,25 @@
#include <unistd.h>
#include <string.h>
#include <limits.h>
#include <errno.h>
#ifdef HAVE_LOCALE_H
#include <locale.h>
#endif
#ifdef WITH_IDN
#ifdef WITH_IDN_SUPPORT
#ifdef WITH_IDNKIT
#include <idn/result.h>
#include <idn/log.h>
#include <idn/resconf.h>
#include <idn/api.h>
#endif
#ifdef WITH_LIBIDN2
#include <idn2.h>
#endif
#endif /* WITH_IDN_SUPPORT */
#include <dns/byaddr.h>
#ifdef DIG_SIGCHASE
#include <dns/callbacks.h>
......@@ -148,18 +155,26 @@ int lookup_counter = 0;
static char servercookie[256];
#ifdef WITH_IDN
static void initialize_idn(void);
static isc_result_t output_filter(isc_buffer_t *buffer,
unsigned int used_org,
isc_boolean_t absolute);
static idn_result_t append_textname(char *name, const char *origin,
size_t namesize);
static void idn_check_result(idn_result_t r, const char *msg);
#define MAXDLEN 256
int idnoptions = 0;
#ifdef WITH_IDN_SUPPORT
static void idn_initialize(void);
static isc_result_t idn_locale_to_ace(const char *from,
char *to,
size_t tolen);
#endif /* WITH_IDN_SUPPORT */
#ifdef WITH_IDN_OUT_SUPPORT
static isc_result_t idn_ace_to_locale(const char *from,
char *to,
size_t tolen);
static isc_result_t output_filter(isc_buffer_t *buffer,
unsigned int used_org,
isc_boolean_t absolute);
#define MAXDLEN 256
#ifdef WITH_IDNKIT
int idnoptions = 0;
#endif
#endif /* WITH_IDN_OUT_SUPPORT */
isc_socket_t *keep = NULL;
isc_sockaddr_t keepaddr;
......@@ -806,7 +821,12 @@ make_empty_lookup(void) {
looknew->sendcookie = ISC_FALSE;
looknew->seenbadcookie = ISC_FALSE;
looknew->badcookie = ISC_TRUE;
#ifdef WITH_IDN
#ifdef WITH_IDN_SUPPORT
looknew->idnin = ISC_TRUE;
#else
looknew->idnin = ISC_FALSE;
#endif
#ifdef WITH_IDN_OUT_SUPPORT
looknew->idnout = ISC_TRUE;
#else
looknew->idnout = ISC_FALSE;
......@@ -955,6 +975,7 @@ clone_lookup(dig_lookup_t *lookold, isc_boolean_t servers) {
}
looknew->ednsneg = lookold->ednsneg;
looknew->mapped = lookold->mapped;
looknew->idnin = lookold->idnin;
looknew->idnout = lookold->idnout;
#ifdef DIG_SIGCHASE
looknew->sigchase = lookold->sigchase;
......@@ -1515,8 +1536,19 @@ setup_system(isc_boolean_t ipv4only, isc_boolean_t ipv6only) {
copy_server_list(lwconf, &server_list);
}
#ifdef WITH_IDN
initialize_idn();
#ifdef HAVE_SETLOCALE
/* Set locale */
(void)setlocale(LC_ALL, "");
#endif
#ifdef WITH_IDN_SUPPORT
idn_initialize();
#endif
#ifdef WITH_IDN_OUT_SUPPORT
/* Set domain name -> text post-conversion filter. */
result = dns_name_settotextfilter(output_filter);
check_result(result, "dns_name_settotextfilter");
#endif
if (keyfile[0] != 0)
......@@ -2343,12 +2375,13 @@ setup_lookup(dig_lookup_t *lookup) {
char store[MXNAME];
char ecsbuf[20];
char cookiebuf[256];
#ifdef WITH_IDN
idn_result_t mr;
char utf8_textname[MXNAME], utf8_origin[MXNAME], idn_textname[MXNAME];
char *origin = NULL;
char *textname = NULL;
#ifdef WITH_IDN_SUPPORT
char idn_origin[MXNAME], idn_textname[MXNAME];
#endif
#ifdef WITH_IDN
#ifdef WITH_IDN_OUT_SUPPORT
result = dns_name_settotextfilter(lookup->idnout ?
output_filter : NULL);
check_result(result, "dns_name_settotextfilter");
......@@ -2381,15 +2414,20 @@ setup_lookup(dig_lookup_t *lookup) {
isc_buffer_init(&lookup->onamebuf, lookup->oname_space,
sizeof(lookup->oname_space));
#ifdef WITH_IDN
/*
* We cannot convert `textname' and `origin' separately.
* `textname' doesn't contain TLD, but local mapping needs
* TLD.
*/
mr = idn_encodename(IDN_LOCALCONV | IDN_DELIMMAP, lookup->textname,
utf8_textname, sizeof(utf8_textname));
idn_check_result(mr, "convert textname to UTF-8");
textname = lookup->textname;
#ifdef WITH_IDN_SUPPORT
if (lookup->idnin) {
result = idn_locale_to_ace(lookup->textname, idn_textname,
sizeof(idn_textname));
check_result(result, "convert textname to IDN encoding");
debug("idn_textname: %s", idn_textname);
textname = idn_textname;
}
#endif
/*
......@@ -2400,17 +2438,8 @@ setup_lookup(dig_lookup_t *lookup) {
* is TRUE or we got a domain line in the resolv.conf file.
*/
if (lookup->new_search) {
#ifdef WITH_IDN
if ((count_dots(utf8_textname) >= ndots) || !usesearch) {
lookup->origin = NULL; /* Force abs lookup */
lookup->done_as_is = ISC_TRUE;
lookup->need_search = usesearch;
} else if (lookup->origin == NULL && usesearch) {
lookup->origin = ISC_LIST_HEAD(search_list);
lookup->need_search = ISC_FALSE;
}
#else
if ((count_dots(lookup->textname) >= ndots) || !usesearch) {
if ((count_dots(textname) >= ndots) || !usesearch)
{
lookup->origin = NULL; /* Force abs lookup */
lookup->done_as_is = ISC_TRUE;
lookup->need_search = usesearch;
......@@ -2418,24 +2447,8 @@ setup_lookup(dig_lookup_t *lookup) {
lookup->origin = ISC_LIST_HEAD(search_list);
lookup->need_search = ISC_FALSE;
}
#endif
}
#ifdef WITH_IDN
if (lookup->origin != NULL) {
mr = idn_encodename(IDN_LOCALCONV | IDN_DELIMMAP,
lookup->origin->origin, utf8_origin,
sizeof(utf8_origin));
idn_check_result(mr, "convert origin to UTF-8");
mr = append_textname(utf8_textname, utf8_origin,
sizeof(utf8_textname));
idn_check_result(mr, "append origin to textname");
}
mr = idn_encodename(idnoptions | IDN_LOCALMAP | IDN_NAMEPREP |
IDN_IDNCONV | IDN_LENCHECK, utf8_textname,
idn_textname, sizeof(idn_textname));
idn_check_result(mr, "convert UTF-8 textname to IDN encoding");
#else
if (lookup->origin != NULL) {
debug("trying origin %s", lookup->origin->origin);
result = dns_message_gettempname(lookup->sendmsg,
......@@ -2443,8 +2456,18 @@ setup_lookup(dig_lookup_t *lookup) {
check_result(result, "dns_message_gettempname");
dns_name_init(lookup->oname, NULL);
/* XXX Helper funct to conv char* to name? */
len = (unsigned int) strlen(lookup->origin->origin);
isc_buffer_init(&b, lookup->origin->origin, len);
origin = lookup->origin->origin;
#ifdef WITH_IDN_SUPPORT
if (lookup->idnin) {
result = idn_locale_to_ace(lookup->origin->origin,
idn_origin, sizeof(idn_origin));
check_result(result, "convert origin to IDN encoding");
debug("trying idn origin %s", idn_origin);
origin = idn_origin;
}
#endif
len = (unsigned int) strlen(origin);
isc_buffer_init(&b, origin, len);
isc_buffer_add(&b, len);
result = dns_name_fromtext(lookup->oname, &b, dns_rootname,
0, &lookup->onamebuf);
......@@ -2454,7 +2477,7 @@ setup_lookup(dig_lookup_t *lookup) {
dns_message_puttempname(lookup->sendmsg,
&lookup->oname);
fatal("'%s' is not in legal name syntax (%s)",
lookup->origin->origin,
origin,
isc_result_totext(result));
}
if (lookup->trace && lookup->trace_root) {
......@@ -2465,8 +2488,8 @@ setup_lookup(dig_lookup_t *lookup) {
dns_fixedname_init(&fixed);
name = dns_fixedname_name(&fixed);
len = (unsigned int) strlen(lookup->textname);
isc_buffer_init(&b, lookup->textname, len);
len = (unsigned int) strlen(textname);
isc_buffer_init(&b, textname, len);
isc_buffer_add(&b, len);
result = dns_name_fromtext(name, &b, NULL, 0, NULL);
if (result == ISC_R_SUCCESS &&
......@@ -2491,28 +2514,17 @@ setup_lookup(dig_lookup_t *lookup) {
}
}
dns_message_puttempname(lookup->sendmsg, &lookup->oname);
} else
#endif
{
} else {
debug("using root origin");
if (lookup->trace && lookup->trace_root)
dns_name_clone(dns_rootname, lookup->name);
else {
#ifdef WITH_IDN
len = (unsigned int) strlen(idn_textname);
isc_buffer_init(&b, idn_textname, len);
len = (unsigned int) strlen(textname);
isc_buffer_init(&b, textname, len);
isc_buffer_add(&b, len);
result = dns_name_fromtext(lookup->name, &b,
dns_rootname, 0,
&lookup->namebuf);
#else
len = (unsigned int) strlen(lookup->textname);
isc_buffer_init(&b, lookup->textname, len);
isc_buffer_add(&b, len);
result = dns_name_fromtext(lookup->name, &b,
dns_rootname, 0,
&lookup->namebuf);
#endif
}
if (result != ISC_R_SUCCESS) {
dns_message_puttempname(lookup->sendmsg,
......@@ -4511,7 +4523,7 @@ destroy_libs(void) {
void * ptr;
dig_message_t *chase_msg;
#endif
#ifdef WITH_IDN
#ifdef WITH_IDN_SUPPORT
isc_result_t result;
#endif
......@@ -4548,7 +4560,7 @@ destroy_libs(void) {
clear_searchlist();
#ifdef WITH_IDN
#ifdef WITH_IDN_SUPPORT
result = dns_name_settotextfilter(NULL);
check_result(result, "dns_name_settotextfilter");
#endif
......@@ -4632,27 +4644,7 @@ destroy_libs(void) {
isc_mem_destroy(&mctx);
}
#ifdef WITH_IDN
static void
initialize_idn(void) {
idn_result_t r;
isc_result_t result;
#ifdef HAVE_SETLOCALE
/* Set locale */
(void)setlocale(LC_ALL, "");
#endif
/* Create configuration context. */
r = idn_nameinit(1);
if (r != idn_success)
fatal("idn api initialization failed: %s",
idn_result_tostring(r));
/* Set domain name -> text post-conversion filter. */
result = dns_name_settotextfilter(output_filter);
check_result(result, "dns_name_settotextfilter");
}
#ifdef WITH_IDN_OUT_SUPPORT
static isc_result_t
output_filter(isc_buffer_t *buffer, unsigned int used_org,
isc_boolean_t absolute)
......@@ -4660,6 +4652,7 @@ output_filter(isc_buffer_t *buffer, unsigned int used_org,
char tmp1[MAXDLEN], tmp2[MAXDLEN];
size_t fromlen, tolen;
isc_boolean_t end_with_dot;
isc_result_t result;
/*
* Copy contents of 'buffer' to 'tmp1', supply trailing dot
......@@ -4668,6 +4661,7 @@ output_filter(isc_buffer_t *buffer, unsigned int used_org,
fromlen = isc_buffer_usedlength(buffer) - used_org;
if (fromlen >= MAXDLEN)
return (ISC_R_SUCCESS);
memmove(tmp1, (char *)isc_buffer_base(buffer) + used_org, fromlen);
end_with_dot = (tmp1[fromlen - 1] == '.') ? ISC_TRUE : ISC_FALSE;
if (absolute && !end_with_dot) {
......@@ -4676,13 +4670,16 @@ output_filter(isc_buffer_t *buffer, unsigned int used_org,
return (ISC_R_SUCCESS);
tmp1[fromlen - 1] = '.';
}
tmp1[fromlen] = '\0';
/*
* Convert contents of 'tmp1' to local encoding.
*/
if (idn_decodename(IDN_DECODE_APP, tmp1, tmp2, MAXDLEN) != idn_success)
result = idn_ace_to_locale(tmp1, tmp2, sizeof(tmp2));
if (result != ISC_R_SUCCESS) {
return (ISC_R_SUCCESS);
}
strlcpy(tmp1, tmp2, MAXDLEN);
/*
......@@ -4702,35 +4699,118 @@ output_filter(isc_buffer_t *buffer, unsigned int used_org,
return (ISC_R_SUCCESS);
}
#endif
static idn_result_t
append_textname(char *name, const char *origin, size_t namesize) {
size_t namelen = strlen(name);
size_t originlen = strlen(origin);
#ifdef WITH_IDN_SUPPORT
#ifdef WITH_IDNKIT
static void
idnkit_check_result(idn_result_t result, const char *msg) {
if (result != idn_success) {
fatal("%s: %s", msg, idn_result_tostring(result));
}
}
static void
idn_initialize(void) {
idn_result_t result;
/* Already absolute? */
if (namelen > 0 && name[namelen - 1] == '.')
return (idn_success);
/* Create configuration context. */
result = idn_nameinit(1);
idnkit_check_result(result, "idnkit api initialization failed");
return (ISC_R_SUCCESS);
}
/* Append dot and origin */
static isc_result_t
idn_locale_to_ace(const char *from, char *to, size_t tolen) {
char utf8_textname[MXNAME];
idn_result_t result;
result = idn_encodename(IDN_LOCALCONV | IDN_DELIMMAP, from,
utf8_textname, sizeof(utf8_textname));
idnkit_check_result(result, "idnkit idn_encodename to utf8 failed");
result = idn_encodename(idnoptions | IDN_LOCALMAP | IDN_NAMEPREP |
IDN_IDNCONV | IDN_LENCHECK,
utf8_textname, to, tolen);
idnkit_check_result(result, "idnkit idn_encodename to idn failed");
return (ISC_R_SUCCESS);
}
if (namelen + 1 + originlen >= namesize)
return (idn_buffer_overflow);
static isc_result_t
idn_ace_to_locale(const char *from, char *to, size_t tolen) {
idn_result_t result;
if (*origin != '.')
name[namelen++] = '.';
(void)strlcpy(name + namelen, origin, namesize - namelen);
return (idn_success);
result = idn_decodename(IDN_DECODE_APP, from, to, tolen);
if (result != idn_success) {
debug("idnkit idn_decodename failed: %s",
idn_result_tostring(result));
return (ISC_R_FAILURE);
}
return (ISC_R_SUCCESS);
}
#endif /* WITH_IDNKIT */
#ifdef WITH_LIBIDN2
static void
idn_check_result(idn_result_t r, const char *msg) {
if (r != idn_success) {
exitcode = 1;
fatal("%s: %s", msg, idn_result_tostring(r));
idn_initialize(void) {
}
static isc_result_t
idn_locale_to_ace(const char *from, char *to, size_t tolen) {
int res;
char *tmp_str = NULL;
res = idn2_lookup_ul(from, &tmp_str, IDN2_NONTRANSITIONAL);
if (res == IDN2_DISALLOWED)
res = idn2_lookup_ul(from, &tmp_str, IDN2_TRANSITIONAL);
if (res == IDN2_OK) {
/* check the length */
if (strlen(tmp_str) >= tolen) {
debug("ACE string is too long");
idn2_free(tmp_str);
return ISC_R_NOSPACE;
}
(void) strlcpy(to, tmp_str, tolen);
idn2_free(tmp_str);
return ISC_R_SUCCESS;
}
fatal("idn2_lookup_ul failed: %s", idn2_strerror(res));
return ISC_R_FAILURE;
}
#ifdef WITH_IDN_OUT_SUPPORT
static isc_result_t
idn_ace_to_locale(const char *from, char *to, size_t tolen) {
int res;
char *tmp_str = NULL;
res = idn2_to_unicode_8zlz(from, &tmp_str,
IDN2_NONTRANSITIONAL|IDN2_NFC_INPUT);
if (res == IDN2_OK) {
/* check the length */
if (strlen(tmp_str) >= tolen) {
debug("encoded ASC string is too long");
idn2_free(tmp_str);
return ISC_R_FAILURE;
}
(void) strncpy(to, tmp_str, tolen);
free(tmp_str);
return ISC_R_SUCCESS;
} else {
debug("idn2_to_unicode_8zlz failed: %s",
idn2_strerror(res));
}
return ISC_R_FAILURE;
}
#endif /* WITH_IDN */
#endif /* WITH_IDN_OUT_SUPPORT */
#endif /* WITH_LIBIDN2 */
#endif /* WITH_IDN_SUPPORT */
#ifdef DIG_SIGCHASE
void
......
......@@ -19,7 +19,7 @@
#include <locale.h>
#endif
#ifdef WITH_IDN
#ifdef WITH_IDNKIT
#include <idn/result.h>
#include <idn/log.h>
#include <idn/resconf.h>
......@@ -723,7 +723,7 @@ parse_args(isc_boolean_t is_batchfile, int argc, char **argv) {
lookup->rdtype != dns_rdatatype_axfr)
lookup->rdtype = rdtype;
lookup->rdtypeset = ISC_TRUE;
#ifdef WITH_IDN
#ifdef WITH_IDNKIT
idnoptions = 0;
#endif
if (rdtype == dns_rdatatype_axfr) {
......@@ -738,7 +738,7 @@ parse_args(isc_boolean_t is_batchfile, int argc, char **argv) {
} else if (rdtype == dns_rdatatype_any) {
if (!lookup->tcp_mode_set)
lookup->tcp_mode = ISC_TRUE;
#ifdef WITH_IDN
#ifdef WITH_IDNKIT
} else if (rdtype == dns_rdatatype_a ||
rdtype == dns_rdatatype_aaaa ||
rdtype == dns_rdatatype_mx) {
......@@ -770,7 +770,7 @@ parse_args(isc_boolean_t is_batchfile, int argc, char **argv) {
if (!lookup->rdtypeset ||
lookup->rdtype != dns_rdatatype_axfr)
lookup->rdtype = dns_rdatatype_any;
#ifdef WITH_IDN
#ifdef WITH_IDNKIT
idnoptions = 0;
#endif
list_type = dns_rdatatype_any;
......@@ -884,7 +884,7 @@ main(int argc, char **argv) {
ISC_LIST_INIT(search_list);
fatalexit = 1;
#ifdef WITH_IDN
#ifdef WITH_IDNKIT
idnoptions = IDN_ASCCHECK;
#endif
......
......@@ -133,6 +133,7 @@ struct dig_lookup {
ednsneg,
mapped,
print_unknown_format,
idnin,
idnout;
#ifdef DIG_SIGCHASE
isc_boolean_t sigchase;
......@@ -295,7 +296,7 @@ extern char *progname;