Gitweb:
http://git.exim.org/exim.git/commitdiff/3c8b357717e895d6dcddc7faa5b7a0eaf4c73417
Commit: 3c8b357717e895d6dcddc7faa5b7a0eaf4c73417
Parent: e8f0fc219ceca2834ee2b6a1a1a9b960ead39a2b
Author: Jeremy Harris <jgh146exb@???>
AuthorDate: Mon Apr 20 16:48:36 2015 +0100
Committer: Jeremy Harris <jgh146exb@???>
CommitDate: Tue Apr 21 15:53:32 2015 +0100
UTF8: MSA downconversions
---
doc/doc-txt/NewStuff | 5 +-
doc/doc-txt/experimental-spec.txt | 12 +++-
src/src/acl.c | 156 +++++++++++++++++++++-------------
src/src/deliver.c | 15 +++-
src/src/functions.h | 3 +-
src/src/globals.c | 1 +
src/src/globals.h | 1 +
src/src/routers/redirect.c | 4 +-
src/src/spool_in.c | 10 ++
src/src/spool_out.c | 9 ++-
src/src/string.c | 46 ++++++++--
src/src/structs.h | 7 ++-
src/src/transports/smtp.c | 74 +++++++++++++---
src/src/utf8.c | 27 ++++++
src/src/verify.c | 28 +++++--
test/confs/4201 | 2 +-
test/confs/4207 | 1 +
test/log/4207 | 9 ++
test/scripts/4200-International/4207 | 24 +++++
test/stderr/4207 | 2 +
test/stdout/4207 | 12 +++
21 files changed, 347 insertions(+), 101 deletions(-)
diff --git a/doc/doc-txt/NewStuff b/doc/doc-txt/NewStuff
index e4bc586..ef6a6de 100644
--- a/doc/doc-txt/NewStuff
+++ b/doc/doc-txt/NewStuff
@@ -23,7 +23,10 @@ Version 4.86
6. A commandline option to write a comment into the logfile.
7. If built with EXPERIMENTAL_SOCKS feature enabled, the smtp transport can
- be configured to make connections via socks5 proxies
+ be configured to make connections via socks5 proxies.
+
+ 8. If built with EXPERIMENTAL_INTERNATIONAL, support is included for
+ the transmission of UTF-8 envelope addresses.
Version 4.85
diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index 819b47f..234bac0 100644
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -1308,12 +1308,22 @@ New expansion operators:
${utf8_localpart_to_alabel:str}
${utf8_localpart_from_alabel:str}
+New "control = utf8_downconvert" ACL modifier,
+sets a flag requiring that addresses are converted to
+a-label form before smtp delivery, for use in a
+Message Submission Agent context. Can also be
+phrased as "control = utf8_downconvert/1" and is
+mandatory. The flag defaults to zero and can be cleared
+by "control = utf8_downconvert/0". The value "-1"
+may also be used, to use a-label for only if the
+destination host does not support SMTPUTF8.
+
Known issues:
- Currently LMTP is not supported.
- DSN unitext handling is not present
- no provision for converting logging from UTF-8
- VRFY and EXPN not handled
- - MSA mode not handled (!)
+ - mua_wrapper not handled
--------------------------------------------------------------
End of file
diff --git a/src/src/acl.c b/src/src/acl.c
index 3f513c3..c1402a0 100644
--- a/src/src/acl.c
+++ b/src/src/acl.c
@@ -181,17 +181,17 @@ that follows! */
enum {
CONTROL_AUTH_UNADVERTISED,
- #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
CONTROL_BMI_RUN,
- #endif
+#endif
CONTROL_DEBUG,
- #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
CONTROL_DKIM_VERIFY,
- #endif
- #ifdef EXPERIMENTAL_DMARC
+#endif
+#ifdef EXPERIMENTAL_DMARC
CONTROL_DMARC_VERIFY,
CONTROL_DMARC_FORENSIC,
- #endif
+#endif
CONTROL_DSCP,
CONTROL_ERROR,
CONTROL_CASEFUL_LOCAL_PART,
@@ -203,11 +203,14 @@ enum {
CONTROL_QUEUE_ONLY,
CONTROL_SUBMISSION,
CONTROL_SUPPRESS_LOCAL_FIXUPS,
- #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
CONTROL_NO_MBOX_UNSPOOL,
- #endif
+#endif
CONTROL_FAKEDEFER,
CONTROL_FAKEREJECT,
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ CONTROL_UTF8_DOWNCONVERT,
+#endif
CONTROL_NO_MULTILINE,
CONTROL_NO_PIPELINING,
CONTROL_NO_DELAY_FLUSH,
@@ -221,17 +224,17 @@ and should be tidied up. */
static uschar *controls[] = {
US"allow_auth_unadvertised",
- #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
US"bmi_run",
- #endif
+#endif
US"debug",
- #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
US"dkim_disable_verify",
- #endif
- #ifdef EXPERIMENTAL_DMARC
+#endif
+#ifdef EXPERIMENTAL_DMARC
US"dmarc_disable_verify",
US"dmarc_enable_forensic",
- #endif
+#endif
US"dscp",
US"error",
US"caseful_local_part",
@@ -243,11 +246,14 @@ static uschar *controls[] = {
US"queue_only",
US"submission",
US"suppress_local_fixups",
- #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
US"no_mbox_unspool",
- #endif
+#endif
US"fakedefer",
US"fakereject",
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ US"utf8_downconvert",
+#endif
US"no_multiline_responses",
US"no_pipelining",
US"no_delay_flush",
@@ -600,26 +606,26 @@ static unsigned int control_forbids[] = {
(unsigned int)
~((1<<ACL_WHERE_CONNECT)|(1<<ACL_WHERE_HELO)), /* allow_auth_unadvertised */
- #ifdef EXPERIMENTAL_BRIGHTMAIL
+#ifdef EXPERIMENTAL_BRIGHTMAIL
0, /* bmi_run */
- #endif
+#endif
0, /* debug */
- #ifndef DISABLE_DKIM
+#ifndef DISABLE_DKIM
(1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)| /* dkim_disable_verify */
- #ifndef DISABLE_PRDR
+# ifndef DISABLE_PRDR
(1<<ACL_WHERE_PRDR)|
- #endif
+# endif
(1<<ACL_WHERE_NOTSMTP_START),
- #endif
+#endif
- #ifdef EXPERIMENTAL_DMARC
+#ifdef EXPERIMENTAL_DMARC
(1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)| /* dmarc_disable_verify */
(1<<ACL_WHERE_NOTSMTP_START),
(1<<ACL_WHERE_DATA)|(1<<ACL_WHERE_NOTSMTP)| /* dmarc_enable_forensic */
(1<<ACL_WHERE_NOTSMTP_START),
- #endif
+#endif
(1<<ACL_WHERE_NOTSMTP)|
(1<<ACL_WHERE_NOTSMTP_START)|
@@ -663,30 +669,34 @@ static unsigned int control_forbids[] = {
(1<<ACL_WHERE_PREDATA)|
(1<<ACL_WHERE_NOTSMTP_START)),
- #ifdef WITH_CONTENT_SCAN
+#ifdef WITH_CONTENT_SCAN
(unsigned int)
~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)| /* no_mbox_unspool */
(1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
// (1<<ACL_WHERE_PRDR)| /* Not allow one user to freeze for all */
(1<<ACL_WHERE_MIME)),
- #endif
+#endif
(unsigned int)
~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)| /* fakedefer */
(1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
- #ifndef DISABLE_PRDR
+#ifndef DISABLE_PRDR
(1<<ACL_WHERE_PRDR)|
- #endif
+#endif
(1<<ACL_WHERE_MIME)),
(unsigned int)
~((1<<ACL_WHERE_MAIL)|(1<<ACL_WHERE_RCPT)| /* fakereject */
(1<<ACL_WHERE_PREDATA)|(1<<ACL_WHERE_DATA)|
- #ifndef DISABLE_PRDR
+#ifndef DISABLE_PRDR
(1<<ACL_WHERE_PRDR)|
- #endif
+#endif
(1<<ACL_WHERE_MIME)),
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ 0, /* utf8_downconvert */
+#endif
+
(1<<ACL_WHERE_NOTSMTP)| /* no_multiline */
(1<<ACL_WHERE_NOTSMTP_START),
@@ -709,37 +719,40 @@ typedef struct control_def {
} control_def;
static control_def controls_list[] = {
- { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED, FALSE },
+ { US"allow_auth_unadvertised", CONTROL_AUTH_UNADVERTISED, FALSE },
#ifdef EXPERIMENTAL_BRIGHTMAIL
- { US"bmi_run", CONTROL_BMI_RUN, FALSE },
+ { US"bmi_run", CONTROL_BMI_RUN, FALSE },
#endif
- { US"debug", CONTROL_DEBUG, TRUE },
+ { US"debug", CONTROL_DEBUG, TRUE },
#ifndef DISABLE_DKIM
- { US"dkim_disable_verify", CONTROL_DKIM_VERIFY, FALSE },
+ { US"dkim_disable_verify", CONTROL_DKIM_VERIFY, FALSE },
#endif
#ifdef EXPERIMENTAL_DMARC
- { US"dmarc_disable_verify", CONTROL_DMARC_VERIFY, FALSE },
- { US"dmarc_enable_forensic", CONTROL_DMARC_FORENSIC, FALSE },
+ { US"dmarc_disable_verify", CONTROL_DMARC_VERIFY, FALSE },
+ { US"dmarc_enable_forensic", CONTROL_DMARC_FORENSIC, FALSE },
#endif
- { US"dscp", CONTROL_DSCP, TRUE },
- { US"caseful_local_part", CONTROL_CASEFUL_LOCAL_PART, FALSE },
- { US"caselower_local_part", CONTROL_CASELOWER_LOCAL_PART, FALSE },
- { US"enforce_sync", CONTROL_ENFORCE_SYNC, FALSE },
- { US"freeze", CONTROL_FREEZE, TRUE },
- { US"no_callout_flush", CONTROL_NO_CALLOUT_FLUSH, FALSE },
- { US"no_delay_flush", CONTROL_NO_DELAY_FLUSH, FALSE },
- { US"no_enforce_sync", CONTROL_NO_ENFORCE_SYNC, FALSE },
- { US"no_multiline_responses", CONTROL_NO_MULTILINE, FALSE },
- { US"no_pipelining", CONTROL_NO_PIPELINING, FALSE },
- { US"queue_only", CONTROL_QUEUE_ONLY, FALSE },
+ { US"dscp", CONTROL_DSCP, TRUE },
+ { US"caseful_local_part", CONTROL_CASEFUL_LOCAL_PART, FALSE },
+ { US"caselower_local_part", CONTROL_CASELOWER_LOCAL_PART, FALSE },
+ { US"enforce_sync", CONTROL_ENFORCE_SYNC, FALSE },
+ { US"freeze", CONTROL_FREEZE, TRUE },
+ { US"no_callout_flush", CONTROL_NO_CALLOUT_FLUSH, FALSE },
+ { US"no_delay_flush", CONTROL_NO_DELAY_FLUSH, FALSE },
+ { US"no_enforce_sync", CONTROL_NO_ENFORCE_SYNC, FALSE },
+ { US"no_multiline_responses", CONTROL_NO_MULTILINE, FALSE },
+ { US"no_pipelining", CONTROL_NO_PIPELINING, FALSE },
+ { US"queue_only", CONTROL_QUEUE_ONLY, FALSE },
#ifdef WITH_CONTENT_SCAN
- { US"no_mbox_unspool", CONTROL_NO_MBOX_UNSPOOL, FALSE },
+ { US"no_mbox_unspool", CONTROL_NO_MBOX_UNSPOOL, FALSE },
#endif
- { US"fakedefer", CONTROL_FAKEDEFER, TRUE },
- { US"fakereject", CONTROL_FAKEREJECT, TRUE },
- { US"submission", CONTROL_SUBMISSION, TRUE },
+ { US"fakedefer", CONTROL_FAKEDEFER, TRUE },
+ { US"fakereject", CONTROL_FAKEREJECT, TRUE },
+ { US"submission", CONTROL_SUBMISSION, TRUE },
{ US"suppress_local_fixups", CONTROL_SUPPRESS_LOCAL_FIXUPS, FALSE },
- { US"cutthrough_delivery", CONTROL_CUTTHROUGH_DELIVERY, FALSE }
+ { US"cutthrough_delivery", CONTROL_CUTTHROUGH_DELIVERY, FALSE },
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ { US"utf8_downconvert", CONTROL_UTF8_DOWNCONVERT, TRUE }
+#endif
};
/* Support data structures for Client SMTP Authorization. acl_verify_csa()
@@ -2080,7 +2093,11 @@ else if (verify_sender_address != NULL)
sender_vaddr = deliver_make_addr(verify_sender_address, TRUE);
#ifdef EXPERIMENTAL_INTERNATIONAL
- sender_vaddr->prop.utf8 = message_smtputf8;
+ if ((sender_vaddr->prop.utf8_msg = message_smtputf8))
+ {
+ sender_vaddr->prop.utf8_downcvt = message_utf8_downconvert == 1;
+ sender_vaddr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+ }
#endif
if (no_details) setflag(sender_vaddr, af_sverify_told);
if (verify_sender_address[0] != 0)
@@ -3377,6 +3394,24 @@ for (; cb != NULL; cb = cb->next)
arg, *log_msgptr);
}
return ERROR;
+
+ #ifdef EXPERIMENTAL_INTERNATIONAL
+ case CONTROL_UTF8_DOWNCONVERT:
+ if (*p == '/')
+ {
+ if (p[1] == '1') { message_utf8_downconvert = 1; p += 2; break; }
+ if (p[1] == '0') { message_utf8_downconvert = 0; p += 2; break; }
+ if (p[1] == '-' && p[2] == '1')
+ { message_utf8_downconvert = -1; p += 3; break; }
+ *log_msgptr = US"bad option value for control=utf8_downconvert";
+ }
+ else
+ {
+ message_utf8_downconvert = 1; break;
+ }
+ return ERROR;
+ #endif
+
}
break;
}
@@ -3390,14 +3425,9 @@ for (; cb != NULL; cb = cb->next)
/* Run the dcc backend. */
rc = dcc_process(&ss);
/* Modify return code based upon the existance of options. */
- while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size))
- != NULL) {
+ while ((ss = string_nextinlist(&list, &sep, big_buffer, big_buffer_size)))
if (strcmpic(ss, US"defer_ok") == 0 && rc == DEFER)
- {
- /* FAIL so that the message is passed to the next ACL */
- rc = FAIL;
- }
- }
+ rc = FAIL; /* FAIL so that the message is passed to the next ACL */
}
break;
#endif
@@ -4382,7 +4412,11 @@ if (where == ACL_WHERE_RCPT)
return DEFER;
}
#ifdef EXPERIMENTAL_INTERNATIONAL
- addr->prop.utf8 = message_smtputf8;
+ if ((addr->prop.utf8_msg = message_smtputf8))
+ {
+ addr->prop.utf8_downcvt = message_utf8_downconvert == 1;
+ addr->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+ }
#endif
deliver_domain = addr->domain;
deliver_localpart = addr->local_part;
diff --git a/src/src/deliver.c b/src/src/deliver.c
index f208f3e..58b9d3a 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -822,7 +822,14 @@ if (log_extra_selector & LX_incoming_interface && sending_ip_address)
/* for the port: string_sprintf("%d", sending_port) */
if ((log_extra_selector & LX_sender_on_delivery) != 0 || msg)
- s = string_append(s, &size, &ptr, 3, US" F=<", sender_address, US">");
+ s = string_append(s, &size, &ptr, 3, US" F=<",
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ testflag(addr, af_utf8_downcvt)
+ ? string_address_utf8_to_alabel(sender_address, NULL)
+ :
+#endif
+ sender_address,
+ US">");
#ifdef EXPERIMENTAL_SRS
if(addr->prop.srs_sender)
@@ -5595,7 +5602,11 @@ if (process_recipients != RECIP_IGNORE)
address_item *new = deliver_make_addr(r->address, FALSE);
new->prop.errors_address = r->errors_to;
#ifdef EXPERIMENTAL_INTERNATIONAL
- new->prop.utf8 = message_smtputf8;
+ if ((new->prop.utf8_msg = message_smtputf8))
+ {
+ new->prop.utf8_downcvt = message_utf8_downconvert == 1;
+ new->prop.utf8_downcvt_maybe = message_utf8_downconvert == -1;
+ }
DEBUG(D_deliver) if (message_smtputf8) debug_printf("utf8\n");
#endif
diff --git a/src/src/functions.h b/src/src/functions.h
index d1ada38..1708b7a 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -422,8 +422,7 @@ extern const uschar *string_printing2(const uschar *, BOOL);
extern uschar *string_split_message(uschar *);
extern uschar *string_unprinting(uschar *);
#ifdef EXPERIMENTAL_INTERNATIONAL
-extern uschar *string_address_alabel_to_utf8(const uschar *, uschar **);
-extern uschar *string_address_utf8_to_alabel(uschar *, uschar **, int *);
+extern uschar *string_address_utf8_to_alabel(const uschar *, uschar **);
extern uschar *string_domain_alabel_to_utf8(const uschar *, uschar **);
extern uschar *string_domain_utf8_to_alabel(const uschar *, uschar **);
extern uschar *string_localpart_alabel_to_utf8(const uschar *, uschar **);
diff --git a/src/src/globals.c b/src/src/globals.c
index 2bf4d0a..a71c80e 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -915,6 +915,7 @@ int message_size = 0;
uschar *message_size_limit = US"50M";
#ifdef EXPERIMENTAL_INTERNATIONAL
BOOL message_smtputf8 = FALSE;
+int message_utf8_downconvert = 0; /* -1 ifneeded; 0 never; 1 always */
#endif
uschar message_subdir[2] = { 0, 0 };
uschar *message_reference = NULL;
diff --git a/src/src/globals.h b/src/src/globals.h
index 7cbf7bf..8aa69bf 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -572,6 +572,7 @@ extern int message_size; /* Size of message */
extern uschar *message_size_limit; /* As it says */
#ifdef EXPERIMENTAL_INTERNATIONAL
extern BOOL message_smtputf8; /* Internationalized mail handling */
+extern int message_utf8_downconvert; /* convert from utf8 */
const extern pcre *regex_UTF8; /* For recognizing SMTPUTF8 settings */
#endif
extern uschar message_subdir[]; /* Subdirectory for messages */
diff --git a/src/src/routers/redirect.c b/src/src/routers/redirect.c
index 8f1c2c3..9e57c8b 100644
--- a/src/src/routers/redirect.c
+++ b/src/src/routers/redirect.c
@@ -452,7 +452,7 @@ while (generated != NULL)
}
#ifdef EXPERIMENTAL_INTERNATIONAL
- next->prop.utf8 = string_is_utf8(next->address)
+ next->prop.utf8_msg = string_is_utf8(next->address)
|| (sender_address && string_is_utf8(sender_address));
#endif
@@ -476,7 +476,7 @@ while (generated != NULL)
debug_printf("gid=unset ");
#ifdef EXPERIMENTAL_INTERNATIONAL
- if (next->prop.utf8) debug_printf("utf8 ");
+ if (next->prop.utf8_msg) debug_printf("utf8 ");
#endif
debug_printf("home=%s\n", next->home_dir);
diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 742f4b5..558d955 100644
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -301,6 +301,7 @@ spam_score_int = NULL;
#if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
message_smtputf8 = FALSE;
+message_utf8_downconvert = 0;
#endif
dsn_ret = 0;
@@ -600,6 +601,15 @@ for (;;)
break;
#endif
+#if defined(EXPERIMENTAL_INTERNATIONAL) && !defined(COMPILE_UTILITY)
+ case 'u':
+ if (Ustrncmp(p, "tf8_downcvt", 11) == 0)
+ message_utf8_downconvert = 1;
+ else if (Ustrncmp(p, "tf8_downcvt_opt", 15) == 0)
+ message_utf8_downconvert = -1;
+ break;
+#endif
+
default: /* Present because some compilers complain if all */
break; /* possibilities are not covered. */
}
diff --git a/src/src/spool_out.c b/src/src/spool_out.c
index 6d22bff..48f27a8 100644
--- a/src/src/spool_out.c
+++ b/src/src/spool_out.c
@@ -246,7 +246,12 @@ if (tls_in.ocsp) fprintf(f, "-tls_ocsp %d\n", tls_in.ocsp);
#endif
#ifdef EXPERIMENTAL_INTERNATIONAL
-if (message_smtputf8) fprintf(f, "-smtputf8\n");
+if (message_smtputf8)
+ {
+ fprintf(f, "-smtputf8\n");
+ if (message_utf8_downconvert)
+ fprintf(f, "-utf8_downcvt%s\n", message_utf8_downconvert < 0 ? "_opt" : "");
+ }
#endif
/* Write the dsn flags to the spool header file */
@@ -508,3 +513,5 @@ return TRUE;
#endif
/* End of spool_out.c */
+/* vi: aw ai sw=2
+*/
diff --git a/src/src/string.c b/src/src/string.c
index e169a9f..c50a347 100644
--- a/src/src/string.c
+++ b/src/src/string.c
@@ -1547,14 +1547,35 @@ static uschar *
string_get_localpart(address_item *addr, uschar *yield, int *sizeptr,
int *ptrptr)
{
-if (testflag(addr, af_include_affixes) && addr->prefix != NULL)
- yield = string_cat(yield, sizeptr, ptrptr, addr->prefix,
- Ustrlen(addr->prefix));
-yield = string_cat(yield, sizeptr, ptrptr, addr->local_part,
- Ustrlen(addr->local_part));
-if (testflag(addr, af_include_affixes) && addr->suffix != NULL)
- yield = string_cat(yield, sizeptr, ptrptr, addr->suffix,
- Ustrlen(addr->suffix));
+uschar * s;
+
+s = addr->prefix;
+if (testflag(addr, af_include_affixes) && s)
+ {
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ if (testflag(addr, af_utf8_downcvt))
+ s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+ yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+ }
+
+s = addr->local_part;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+if (testflag(addr, af_utf8_downcvt))
+ s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+
+s = addr->suffix;
+if (testflag(addr, af_include_affixes) && s)
+ {
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ if (testflag(addr, af_utf8_downcvt))
+ s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+ yield = string_cat(yield, sizeptr, ptrptr, s, Ustrlen(s));
+ }
+
return yield;
}
@@ -1615,10 +1636,15 @@ else
{
if (addr->local_part != NULL)
{
+ const uschar * s;
yield = string_get_localpart(addr, yield, &size, &ptr);
yield = string_cat(yield, &size, &ptr, US"@", 1);
- yield = string_cat(yield, &size, &ptr, addr->domain,
- Ustrlen(addr->domain) );
+ s = addr->domain;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ if (testflag(addr, af_utf8_downcvt))
+ s = string_localpart_utf8_to_alabel(s, NULL);
+#endif
+ yield = string_cat(yield, &size, &ptr, s, Ustrlen(s) );
}
else
{
diff --git a/src/src/structs.h b/src/src/structs.h
index 99d65cf..c181f3f 100644
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -460,7 +460,9 @@ typedef struct address_item_propagated {
uschar *srs_sender; /* Change return path when delivering */
#endif
#ifdef EXPERIMENTAL_INTERNATIONAL
- BOOL utf8; /* requires SMTPUTF8 processing */
+ BOOL utf8_msg:1; /* requires SMTPUTF8 processing */
+ BOOL utf8_downcvt:1; /* mandatory downconvert on delivery */
+ BOOL utf8_downcvt_maybe:1; /* optional downconvert on delivery */
#endif
} address_item_propagated;
@@ -500,6 +502,9 @@ typedef struct address_item_propagated {
#ifdef EXPERIMENTAL_DANE
# define af_dane_verified 0x20000000 /* TLS cert verify done with DANE */
#endif
+#ifdef EXPERIMENTAL_INTERNATIONAL
+# define af_utf8_downcvt 0x40000000 /* downconvert was done for delivery */
+#endif
/* These flags must be propagated when a child is created */
diff --git a/src/src/transports/smtp.c b/src/src/transports/smtp.c
index 65bb1de..7537e6e 100644
--- a/src/src/transports/smtp.c
+++ b/src/src/transports/smtp.c
@@ -1366,6 +1366,7 @@ BOOL prdr_offered = FALSE;
BOOL prdr_active;
#endif
#ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_needed = FALSE;
BOOL utf8_offered = FALSE;
#endif
BOOL dsn_all_lasthop = TRUE;
@@ -1642,10 +1643,17 @@ goto SEND_QUIT;
#endif
#ifdef EXPERIMENTAL_INTERNATIONAL
- utf8_offered = esmtp
- && addrlist->prop.utf8
- && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
- PCRE_EOPT, NULL, 0) >= 0;
+ if (addrlist->prop.utf8_msg)
+ {
+ utf8_needed = !addrlist->prop.utf8_downcvt
+ && !addrlist->prop.utf8_downcvt_maybe;
+ DEBUG(D_transport) if (!utf8_needed) debug_printf("utf8: %s downconvert\n",
+ addrlist->prop.utf8_downcvt ? "mandatory" : "optional");
+
+ utf8_offered = esmtp
+ && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+ PCRE_EOPT, NULL, 0) >= 0;
+ }
#endif
}
@@ -1862,10 +1870,10 @@ if (continue_hostname == NULL
#endif
#ifdef EXPERIMENTAL_INTERNATIONAL
- utf8_offered = esmtp
- && addrlist->prop.utf8
- && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
- PCRE_EOPT, NULL, 0) >= 0;
+ if (addrlist->prop.utf8_msg)
+ utf8_offered = esmtp
+ && pcre_exec(regex_UTF8, NULL, CS buffer, Ustrlen(buffer), 0,
+ PCRE_EOPT, NULL, 0) >= 0;
#endif
/* Note if the server supports DSN */
@@ -1896,7 +1904,7 @@ setting_up = FALSE;
#ifdef EXPERIMENTAL_INTERNATIONAL
/* If this is an international message we need the host to speak SMTPUTF8 */
-if (addrlist->prop.utf8 && !utf8_offered)
+if (utf8_needed && !utf8_offered)
{
errno = ERRNO_UTF8_FWD;
goto RESPONSE_FAILED;
@@ -1980,7 +1988,7 @@ if (prdr_offered)
#endif
#ifdef EXPERIMENTAL_INTERNATIONAL
-if (addrlist->prop.utf8)
+if (addrlist->prop.utf8_msg && !addrlist->prop.utf8_downcvt && utf8_offered)
sprintf(CS p, " SMTPUTF8"), p += 9;
#endif
@@ -2037,8 +2045,31 @@ buffer. */
pending_MAIL = TRUE; /* The block starts with MAIL */
-rc = smtp_write_command(&outblock, smtp_use_pipelining,
- "MAIL FROM:<%s>%s\r\n", return_path, buffer);
+ {
+ uschar * s = return_path;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ uschar * errstr = NULL;
+
+ /* If we must downconvert, do the from-address here. Remember we had to
+ for the to-addresses (done below), and also (ugly) for re-doing when building
+ the delivery log line. */
+
+ if (addrlist->prop.utf8_msg && (addrlist->prop.utf8_downcvt || !utf8_offered))
+ {
+ if (s = string_address_utf8_to_alabel(return_path, &errstr), errstr)
+ {
+ set_errno(addrlist, ERRNO_EXPANDFAIL, errstr, DEFER, FALSE, NULL);
+ yield = ERROR;
+ goto SEND_QUIT;
+ }
+ setflag(addrlist, af_utf8_downcvt);
+ }
+#endif
+
+ rc = smtp_write_command(&outblock, smtp_use_pipelining,
+ "MAIL FROM:<%s>%s\r\n", s, buffer);
+ }
+
mail_command = string_copy(big_buffer); /* Save for later error message */
switch(rc)
@@ -2080,6 +2111,7 @@ for (addr = first_addr;
{
int count;
BOOL no_flush;
+ uschar * rcpt_addr;
addr->dsn_aware = smtp_use_dsn ? dsn_support_yes : dsn_support_no;
@@ -2124,8 +2156,24 @@ for (addr = first_addr;
yield as OK, because this error can often mean that there is a problem with
just one address, so we don't want to delay the host. */
+ rcpt_addr = transport_rcpt_address(addr, tblock->rcpt_include_affixes);
+
+#ifdef EXPERIMENTAL_INTERNATIONAL
+ {
+ uschar * dummy_errstr;
+ if ( testflag(addrlist, af_utf8_downcvt)
+ && (rcpt_addr = string_address_utf8_to_alabel(rcpt_addr, &dummy_errstr),
+ dummy_errstr
+ ) )
+ {
+ errno = ERRNO_EXPANDFAIL;
+ goto SEND_FAILED;
+ }
+ }
+#endif
+
count = smtp_write_command(&outblock, no_flush, "RCPT TO:<%s>%s%s\r\n",
- transport_rcpt_address(addr, tblock->rcpt_include_affixes), igquotstr, buffer);
+ rcpt_addr, igquotstr, buffer);
if (count < 0) goto SEND_FAILED;
if (count > 0)
diff --git a/src/src/utf8.c b/src/src/utf8.c
index 6bc0c2e..09ebdf1 100644
--- a/src/src/utf8.c
+++ b/src/src/utf8.c
@@ -24,6 +24,7 @@ return FALSE;
/**************************************************/
/* Domain conversions */
+/* the *err string pointer should be null before the call */
uschar *
string_domain_utf8_to_alabel(const uschar * utf8, uschar ** err)
@@ -68,6 +69,7 @@ return s;
/**************************************************/
/* localpart conversions */
+/* the *err string pointer should be null before the call */
uschar *
@@ -126,6 +128,31 @@ return res;
}
+/**************************************************/
+/* whole address conversion */
+/* the *err string pointer should be null before the call */
+
+uschar *
+string_address_utf8_to_alabel(const uschar * utf8, uschar ** err)
+{
+const uschar * s;
+uschar * l;
+uschar * d;
+
+for (s = utf8; *s; s++)
+ if (*s == '@')
+ {
+ l = string_copyn(utf8, s - utf8);
+ return (l = string_localpart_utf8_to_alabel(l, err), err && *err)
+ || (d = string_domain_utf8_to_alabel(++s, err), err && *err)
+ ? NULL
+ : string_sprintf("%s@%s", l, d);
+ }
+return string_localpart_utf8_to_alabel(utf8, err);
+}
+
+
+
/*************************************************
* Report the library versions. *
*************************************************/
diff --git a/src/src/verify.c b/src/src/verify.c
index 4e9b563..28013fa 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -173,6 +173,9 @@ dbdata_callout_cache new_domain_record;
dbdata_callout_cache_address new_address_record;
host_item *host;
time_t callout_start_time;
+#ifdef EXPERIMENTAL_INTERNATIONAL
+BOOL utf8_offered = FALSE;
+#endif
new_domain_record.result = ccache_unknown;
new_domain_record.postmaster_result = ccache_unknown;
@@ -921,22 +924,35 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount.
}
#ifdef EXPERIMENTAL_INTERNATIONAL
- else if ( addr->prop.utf8
+ else if ( addr->prop.utf8_msg
+ && !addr->prop.utf8_downcvt
&& !( esmtp
&& ( regex_UTF8
|| ( (regex_UTF8 = regex_must_compile(
US"\\n250[\\s\\-]SMTPUTF8(\\s|\\n|$)", FALSE, TRUE)),
TRUE
) )
- && pcre_exec(regex_UTF8, NULL, CS responsebuffer,
- Ustrlen(responsebuffer), 0, PCRE_EOPT, NULL, 0) >= 0
- ) )
+ && ( (utf8_offered = pcre_exec(regex_UTF8, NULL,
+ CS responsebuffer, Ustrlen(responsebuffer),
+ 0, PCRE_EOPT, NULL, 0) >= 0)
+ || addr->prop.utf8_downcvt_maybe
+ ) ) )
{
HDEBUG(D_acl|D_v) debug_printf("utf8 required but not offered\n");
errno = ERRNO_UTF8_FWD;
setflag(addr, af_verify_nsfail);
done = FALSE;
}
+ else if ( addr->prop.utf8_msg
+ && (addr->prop.utf8_downcvt || !utf8_offered)
+ && (from_address = string_address_utf8_to_alabel(from_address,
+ &addr->message), addr->message)
+ )
+ {
+ errno = ERRNO_EXPANDFAIL;
+ setflag(addr, af_verify_nsfail);
+ done = FALSE;
+ }
#endif
/* If we haven't authenticated, but are required to, give up. */
@@ -958,7 +974,7 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount.
/* Send the MAIL command */
(smtp_write_command(&outblock, FALSE,
#ifdef EXPERIMENTAL_INTERNATIONAL
- addr->prop.utf8
+ addr->prop.utf8_msg
? "MAIL FROM:<%s>%s SMTPUTF8\r\n"
:
#endif
@@ -1049,7 +1065,7 @@ can do it there for the non-rcpt-verify case. For this we keep an addresscount.
smtp_write_command(&outblock, FALSE,
#ifdef EXPERIMENTAL_INTERNATIONAL
- addr->prop.utf8
+ addr->prop.utf8_msg
? "MAIL FROM:<%s> SMTPUTF8\r\n"
:
#endif
diff --git a/test/confs/4201 b/test/confs/4201
index b34c7c1..b1fb7a6 100644
--- a/test/confs/4201
+++ b/test/confs/4201
@@ -39,7 +39,7 @@ begin acl
check_recipient:
accept hosts = :
accept domains = +local_domains
- local_parts = ^user.*\$
+ local_parts = ^(xn--)?user.*\$
deny message = relay not permitted
.else
diff --git a/test/confs/4207 b/test/confs/4207
new file mode 120000
index 0000000..73a348f
--- /dev/null
+++ b/test/confs/4207
@@ -0,0 +1 @@
+4201
\ No newline at end of file
diff --git a/test/log/4207 b/test/log/4207
new file mode 100644
index 0000000..e1dd4f7
--- /dev/null
+++ b/test/log/4207
@@ -0,0 +1,9 @@
+1999-03-02 09:44:33 exim x.yz daemon started: pid=pppp, no queue runs, listening for SMTP on port 1225
+1999-03-02 09:44:33 10HmaX-0005vi-00 <= 他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com U=CALLER P=utf8local-esmtp S=sss for user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex
+1999-03-02 09:44:33 10HmaY-0005vi-00 <= xn--ihqwcrb4cv8a8dqg056pqjye@??? H=localhost (the.local.host.name) [127.0.0.1] P=esmtp S=sss id=E10HmaX-0005vi-00@??? for xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@???
+1999-03-02 09:44:33 10HmaX-0005vi-00 => xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@???- <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex> F=<xn--ihqwcrb4cv8a8dqg056pqjye@???> R=rmt T=rmt_smtp H=127.0.0.1 [127.0.0.1] C="250 OK id=10HmaY-0005vi-00"
+1999-03-02 09:44:33 10HmaX-0005vi-00 Completed
+1999-03-02 09:44:33 Start queue run: pid=pppp -qqff
+1999-03-02 09:44:33 10HmaY-0005vi-00 => :blackhole: <xn--user.-f99s29a80cg5i8xgv8fnb734dq4gv6av8eczab60f5jch09a5ea085a0marwd373e180hea90e@???> R=localuser
+1999-03-02 09:44:33 10HmaY-0005vi-00 Completed
+1999-03-02 09:44:33 End queue run: pid=pppp -qqff
diff --git a/test/scripts/4200-International/4207 b/test/scripts/4200-International/4207
new file mode 100644
index 0000000..66e77f0
--- /dev/null
+++ b/test/scripts/4200-International/4207
@@ -0,0 +1,24 @@
+# Internationalised mail: control = utf8_downconvert
+# Exim test configuration 4207
+#
+exim -DSERVER=server -bd -oX PORT_D
+****
+#
+#
+# utf-8 from, mandatory downconvert
+exim -bs -odi -DCONTROL="control=utf8_downconvert"
+EHLO client.bh
+MAIL FROM: <他们为什么不说中文@hebrew.למההםפשוטלאמדבריםעברית.com> SMTPUTF8
+RCPT TO: <user.세계의모든사람들이한국어를이해한다면얼마나좋을까@test.ex>
+DATA
+Subject: test
+
+body
+.
+QUIT
+****
+#
+killdaemon
+exim -DSERVER=server -qqff
+****
+no_msglog_check
diff --git a/test/stderr/4207 b/test/stderr/4207
new file mode 100644
index 0000000..045fadc
--- /dev/null
+++ b/test/stderr/4207
@@ -0,0 +1,2 @@
+
+******** SERVER ********
diff --git a/test/stdout/4207 b/test/stdout/4207
new file mode 100644
index 0000000..94a4e18
--- /dev/null
+++ b/test/stdout/4207
@@ -0,0 +1,12 @@
+220 the.local.host.name ESMTP Exim x.yz Tue, 2 Mar 1999 09:44:33 +0000
+250-the.local.host.name Hello CALLER at client.bh
+250-SIZE 52428800
+250-8BITMIME
+250-PIPELINING
+250-SMTPUTF8
+250 HELP
+250 OK
+250 Accepted
+354 Enter message, ending with "." on a line by itself
+250 OK id=10HmaX-0005vi-00
+221 the.local.host.name closing connection
