The attached patch set is a draft implementation of multi-line macros.
By this I mean macros which can expand into several lines.
The intention of multi-line macros (mlm) is to alleviate the maintenance
problem of having repeated snippets of identical config.
So for example:
########################################################################
procmail_primary_host_comp_prefix:
domains = $primary_hostname
local_part_prefix = "*+"
driver = accept
check_local_user
transport = procmail_pipe
headers_add = Delivered-To: $local_part@$domain
procmail_primary_host_opt_suffix:
domains = $primary_hostname
local_part_suffix = "+*"
local_part_suffix_optional
driver = accept
check_local_user
transport = procmail_pipe
headers_add = Delivered-To: $local_part@$domain
# Other routers here
#
# .
# .
# .
#
procmail_all_local_domains_comp_prefix:
local_part_prefix = "*+"
driver = accept
check_local_user
transport = procmail_pipe
headers_add = Delivered-To: $local_part@$domain
procmail_all_local_domains_opt_suffix:
local_part_suffix = "+*"
local_part_suffix_optional
driver = accept
check_local_user
transport = procmail_pipe
headers_add = Delivered-To: $local_part@$domain
########################################################################
would now be:
########################################################################
ROUTER_PROCMAIL_BODY_COMMON=driver = accept\\
check_local_user\\
transport = procmail_pipe\\
headers_add = Delivered-To: $local_part@
$domain
procmail_primary_host_comp_prefix:
domains = $primary_hostname
local_part_prefix = "*+"
ROUTER_PROCMAIL_BODY_COMMON
procmail_primary_host_opt_suffix:
domains = $primary_hostname
local_part_suffix = "+*"
local_part_suffix_optional
ROUTER_PROCMAIL_BODY_COMMON
# Other routers here
#
# .
# .
# .
#
procmail_all_local_domains_comp_prefix:
local_part_prefix = "*+"
ROUTER_PROCMAIL_BODY_COMMON
procmail_all_local_domains_opt_suffix:
local_part_suffix = "+*"
local_part_suffix_optional
ROUTER_PROCMAIL_BODY_COMMON
########################################################################
"What's the problem?" you say, "just use includes". But this is an
unwelcome overhead in our environment where Exim is being run from
xinetd in multiple private virtual servers. It also makes for an
unhelpful proliferation of separate files.
The first patch, mlm-00-base.patch, contains the multi-line macro
functionality.
The second patch, mlm-01-dump_preprocessed_config.patch, spits out the
preprocessed config file for debugging purposes
The third patch, mlm-02-dprintfs.patch, adds trace statements.
Any comments are welcome as to whether the idea of multi-line macros is
generically useful (i.e. will some implementation make it in?) and if
so, how could this patch could be tidied up, improved or re-implemented.
Regards,
Jonathan Sambrook
Index: exim-4.52/src/readconf.c
===================================================================
--- exim-4.52.orig/src/readconf.c 2005-07-01 12:09:15.%N +0100
+++ exim-4.52/src/readconf.c 2005-09-14 18:09:58.%N +0100
@@ -47,6 +47,9 @@
static config_file_item *config_file_stack = NULL; /* For includes */
+static uschar *expanding_multi_line_macro = NULL; /* For processing multi-line macros */
+static int expanding_multi_line_macro_index = 0;
+
static uschar *syslog_facility_str = NULL;
static uschar next_section[24];
static uschar time_buffer[24];
@@ -465,7 +468,9 @@
read_macro_assignment(uschar *s)
{
uschar name[64];
+uschar *p;
int namelen = 0;
+int count = 0;
BOOL redef = FALSE;
macro_item *m;
macro_item *mlast = NULL;
@@ -553,15 +558,16 @@
m->command_line = FALSE;
}
+/* Check for multi-line expansion */
+
+if (Ustrchr(s, '\n') != NULL)
+ m->multi_line = TRUE;
+
/* Set the value of the new or redefined macro */
m->replacement = string_copy(s);
}
-
-
-
-
/*************************************************
* Read configuration line *
*************************************************/
@@ -583,134 +589,196 @@
Returns: a pointer to the first non-blank in the line,
or NULL if eof or end of section is reached
*/
-
static uschar *
get_config_line(void)
{
int startoffset = 0; /* To first non-blank char in logical line */
int len = 0; /* Of logical line so far */
int newlen;
+int section_name_len;
uschar *s, *ss;
macro_item *m;
-BOOL macro_found;
+BOOL macro_found = FALSE;
+BOOL parsing_multi_line_macro = FALSE;
+
+/* Delayed reclamation of multi-line macro resources */
+if (expanding_multi_line_macro_index == -1)
+ {
+ expanding_multi_line_macro_index = 0;
+ expanding_multi_line_macro = NULL;
+ }
/* Loop for handling continuation lines, skipping comments, and dealing with
.include files. */
for (;;)
{
- if (Ufgets(big_buffer+len, big_buffer_size-len, config_file) == NULL)
+ BOOL multi_line_macro_found = FALSE;
+ if (expanding_multi_line_macro)
+ {
+ /* Skip already processed lines */
+ int count;
+ uschar * p = expanding_multi_line_macro;
+ for (count= 0; count != expanding_multi_line_macro_index; ++count)
+ p += Ustrlen (p) + 1;
+
+ ss = p;
+
+ /* Set index to -1 to flag delayed resource reclamation */
+ if (*(p + Ustrlen(p) + 2) == 0)
+ expanding_multi_line_macro_index = -1;
+ else
+ ++expanding_multi_line_macro_index;
+ }
+ else
{
- if (config_file_stack != NULL) /* EOF inside .include */
+ if (Ufgets(big_buffer+len, big_buffer_size-len, config_file) == NULL)
{
- (void)fclose(config_file);
- config_file = config_file_stack->file;
- config_filename = config_file_stack->filename;
- config_lineno = config_file_stack->lineno;
- config_file_stack = config_file_stack->next;
- continue;
- }
+ if (config_file_stack != NULL) /* EOF inside .include */
+ {
+ (void)fclose(config_file);
+ config_file = config_file_stack->file;
+ config_filename = config_file_stack->filename;
+ config_lineno = config_file_stack->lineno;
+ config_file_stack = config_file_stack->next;
+ continue;
+ }
- /* EOF at top level */
+ /* EOF at top level */
- if (cstate_stack_ptr >= 0)
- log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN,
- "Unexpected end of configuration file: .endif missing");
+ if (cstate_stack_ptr >= 0)
+ log_write(0, LOG_PANIC_DIE|LOG_CONFIG_IN,
+ "Unexpected end of configuration file: .endif missing");
- if (len != 0) break; /* EOF after continuation */
- next_section[0] = 0; /* EOF at start of logical line */
- return NULL;
- }
+ if (len != 0) break; /* EOF after continuation */
+ next_section[0] = 0; /* EOF at start of logical line */
+ return NULL;
+ }
- config_lineno++;
- newlen = len + Ustrlen(big_buffer + len);
+ config_lineno++;
+ newlen = len + Ustrlen(big_buffer + len);
- /* Handle pathologically long physical lines - yes, it did happen - by
- extending big_buffer at this point. The code also copes with very long
- logical lines. */
-
- while (newlen == big_buffer_size - 1 && big_buffer[newlen - 1] != '\n')
- {
- uschar *newbuffer;
- big_buffer_size += BIG_BUFFER_SIZE;
- newbuffer = store_malloc(big_buffer_size);
-
- /* This use of strcpy is OK because we know that the string in the old
- buffer is shorter than the new buffer. */
-
- Ustrcpy(newbuffer, big_buffer);
- store_free(big_buffer);
- big_buffer = newbuffer;
- if (Ufgets(big_buffer+newlen, big_buffer_size-newlen, config_file) == NULL)
- break;
- newlen += Ustrlen(big_buffer + newlen);
- }
+ /* Handle pathologically long physical lines - yes, it did happen - by
+ extending big_buffer at this point. The code also copes with very long
+ logical lines. */
- /* Find the true start of the physical line - leading spaces are always
- ignored. */
+ while (newlen == big_buffer_size - 1 && big_buffer[newlen - 1] != '\n')
+ {
+ uschar *newbuffer;
+ big_buffer_size += BIG_BUFFER_SIZE;
+ newbuffer = store_malloc(big_buffer_size);
- ss = big_buffer + len;
- while (isspace(*ss)) ss++;
+ /* This use of strcpy is OK because we know that the string in the old
+ buffer is shorter than the new buffer. */
- /* Process the physical line for macros. If this is the start of the logical
- line, skip over initial text at the start of the line if it starts with an
- upper case character followed by a sequence of name characters and an equals
- sign, because that is the definition of a new macro, and we don't do
- replacement therein. */
+ Ustrcpy(newbuffer, big_buffer);
+ store_free(big_buffer);
+ big_buffer = newbuffer;
+ if (Ufgets(big_buffer+newlen, big_buffer_size-newlen, config_file) == NULL)
+ break;
+ newlen += Ustrlen(big_buffer + newlen);
+ }
- s = ss;
- if (len == 0 && isupper(*s))
- {
- while (isalnum(*s) || *s == '_') s++;
- while (isspace(*s)) s++;
- if (*s != '=') s = ss; /* Not a macro definition */
- }
+ /* Find the true start of the physical line - leading spaces are always
+ ignored. */
- /* For each defined macro, scan the line (from after XXX= if present),
- replacing all occurrences of the macro. */
+ ss = big_buffer + len;
+ while (isspace(*ss)) ss++;
- macro_found = FALSE;
- for (m = macros; m != NULL; m = m->next)
- {
- uschar *p, *pp;
- uschar *t = s;
+ /* Process the physical line for macros. If this is the start of the logical
+ line, skip over initial text at the start of the line if it starts with an
+ upper case character followed by a sequence of name characters and an equals
+ sign, because that is the definition of a new macro, and we don't do
+ replacement therein. */
- while ((p = Ustrstr(t, m->name)) != NULL)
+ s = ss;
+ if (len == 0 && isupper(*s))
{
- int moveby;
- int namelen = Ustrlen(m->name);
- int replen = Ustrlen(m->replacement);
+ while (isalnum(*s) || *s == '_') s++;
+ while (isspace(*s)) s++;
+ if (*s != '=') s = ss; /* Not a macro definition */
+ }
- /* Expand the buffer if necessary */
+ if (!parsing_multi_line_macro)
+ {
+ /* For each defined macro, scan the line (from after XXX= if present),
+ replacing all occurrences of the macro. */
- while (newlen - namelen + replen + 1 > big_buffer_size)
- {
- int newsize = big_buffer_size + BIG_BUFFER_SIZE;
- uschar *newbuffer = store_malloc(newsize);
- memcpy(newbuffer, big_buffer, newlen + 1);
- p = newbuffer + (p - big_buffer);
- s = newbuffer + (s - big_buffer);
- ss = newbuffer + (ss - big_buffer);
- t = newbuffer + (t - big_buffer);
- big_buffer_size = newsize;
- store_free(big_buffer);
- big_buffer = newbuffer;
+ macro_found = FALSE;
+ BOOL expand_again = TRUE;
+ while (expand_again)
+ {
+ expand_again = FALSE;
+ for (m = macros; m != NULL; m = m->next)
+ {
+ uschar *p, *pp;
+ uschar *t = s;
+
+ while ((p = Ustrstr(t, m->name)) != NULL)
+ {
+ int moveby;
+ int namelen = Ustrlen(m->name);
+ int replen = Ustrlen(m->replacement);
+
+ /* Expand the buffer if necessary */
+
+ while (newlen - namelen + replen + 1 > big_buffer_size)
+ {
+ int newsize = big_buffer_size + BIG_BUFFER_SIZE;
+ uschar *newbuffer = store_malloc(newsize);
+ memcpy(newbuffer, big_buffer, newlen + 1);
+ p = newbuffer + (p - big_buffer);
+ s = newbuffer + (s - big_buffer);
+ ss = newbuffer + (ss - big_buffer);
+ t = newbuffer + (t - big_buffer);
+ big_buffer_size = newsize;
+ store_free(big_buffer);
+ big_buffer = newbuffer;
+ }
+
+ /* Shuffle the remaining characters up or down in the buffer before
+ copying in the replacement text. Don't rescan the replacement for this
+ same macro. */
+
+ pp = p + namelen;
+ moveby = replen - namelen;
+ if (moveby != 0)
+ {
+ memmove(p + replen, pp, (big_buffer + newlen) - pp + 1);
+ newlen += moveby;
+ }
+ Ustrncpy(p, m->replacement, replen);
+ t = p + replen;
+ macro_found = TRUE;
+ if (m->multi_line)
+ {
+ expand_again = TRUE;
+ multi_line_macro_found = TRUE;
+ }
+ }
+ }
+ }
+
+ /* If at least one multi-line macro was expanded, special handling is
+ * necessary */
+ if (multi_line_macro_found)
+ {
+ uschar *p;
+
+ /* Stash copy of lines */
+ expanding_multi_line_macro = store_get (newlen + 2);
+ Ustrncpy (expanding_multi_line_macro, ss, newlen);
+ expanding_multi_line_macro_index = 0;
+
+ /* Substitute \0 for \n */
+ for (p = expanding_multi_line_macro; *p; ++p)
+ if (*p == '\n')
+ *p = 0;
+
+ /* Indicate end of lines with extra \0 */
+ *(p+1) = 0;
+ return get_config_line();
}
-
- /* Shuffle the remaining characters up or down in the buffer before
- copying in the replacement text. Don't rescan the replacement for this
- same macro. */
-
- pp = p + namelen;
- moveby = replen - namelen;
- if (moveby != 0)
- {
- memmove(p + replen, pp, (big_buffer + newlen) - pp + 1);
- newlen += moveby;
- }
- Ustrncpy(p, m->replacement, replen);
- t = p + replen;
- macro_found = TRUE;
}
}
@@ -825,36 +893,54 @@
continue;
}
- /* If this is the start of the logical line, remember where the non-blank
- data starts. Otherwise shuffle down continuation lines to remove leading
- white space. */
-
- if (len == 0)
- startoffset = ss - big_buffer;
+ if (expanding_multi_line_macro)
+ /* NYI recursive multi-line macros */
+ break;
else
{
- s = big_buffer + len;
- if (ss > s)
+ /* If this is the start of the logical line, remember where the non-blank
+ data starts. Otherwise shuffle down continuation lines to remove leading
+ white space. */
+
+ if (len == 0)
+ startoffset = ss - big_buffer;
+ else
{
- memmove(s, ss, (newlen - len) - (ss - s) + 1);
- newlen -= ss - s;
+ s = big_buffer + len;
+ if (ss > s)
+ {
+ memmove(s, ss, (newlen - len) - (ss - s) + 1);
+ newlen -= ss - s;
+ }
}
- }
- /* Accept the new addition to the line. Remove trailing white space. */
+ /* Accept the new addition to the line. Remove trailing white space. */
- len = newlen;
- while (len > 0 && isspace(big_buffer[len-1])) len--;
- big_buffer[len] = 0;
+ len = newlen;
+ while (len > 0 && isspace(big_buffer[len-1])) len--;
+ big_buffer[len] = 0;
- /* We are done if the line does not end in backslash and contains some data.
- Empty logical lines are ignored. For continuations, remove the backslash and
- go round the loop to read the continuation line. */
+ /* We are done if the line does not end in backslash and contains some data.
+ Empty logical lines are ignored. For continuations, remove the backslash and
+ go round the loop to read the continuation line. Multi-line macros have two
+ trailing backslashes; the first is replaced with a \n and the second is
+ treated as a continuation. */
- if (len > 0)
- {
- if (big_buffer[len-1] != '\\') break; /* End of logical line */
- big_buffer[--len] = 0; /* Remove backslash */
+ if (len > 0)
+ {
+ if (big_buffer[len-1] != '\\')
+ {
+ parsing_multi_line_macro = FALSE;
+ break; /* End of logical line */
+ }
+ big_buffer[--len] = 0; /* Remove backslash */
+
+ if (big_buffer[len-1] == '\\') /* Multi-line macro */
+ {
+ parsing_multi_line_macro = TRUE;
+ big_buffer[len-1] = '\n';
+ }
+ }
}
} /* Loop for reading multiple physical lines */
@@ -864,15 +950,21 @@
next_section, truncate it. It will be unrecognized later, because all the known
section names do fit. Leave space for pluralizing. */
-s = big_buffer + startoffset; /* First non-space character */
-if (strncmpic(s, US"begin ", 6) == 0)
+if (expanding_multi_line_macro)
+ s = ss;
+else
{
- s += 6;
- while (isspace(*s)) s++;
- if (big_buffer + len - s > sizeof(next_section) - 2)
- s[sizeof(next_section) - 2] = 0;
- Ustrcpy(next_section, s);
- return NULL;
+ s = big_buffer + startoffset; /* First non-space character */
+
+ if (strncmpic(s, US"begin ", 6) == 0)
+ {
+ s += 6;
+ while (isspace(*s)) s++;
+ if (big_buffer + len - s > sizeof(next_section) - 2)
+ s[sizeof(next_section) - 2] = 0;
+ Ustrcpy(next_section, s);
+ return NULL;
+ }
}
/* Return the first non-blank character. */
Index: exim-4.52/src/readconf.c
===================================================================
--- exim-4.52.orig/src/readconf.c 2005-09-14 17:33:19.%N +0100
+++ exim-4.52/src/readconf.c 2005-09-14 17:53:46.%N +0100
@@ -568,6 +568,66 @@
m->replacement = string_copy(s);
}
+/****************************************************
+* Dump a copy of read config *
+****************************************************/
+
+static void
+dump_preprocessed_config(uschar *s)
+{
+/* Preprocessed config file */
+static BOOL truncated_ppconfig = FALSE;
+
+uschar * ppconfig_file = "/tmp/exim.pp.conf";
+
+if (!truncated_ppconfig)
+ unlink (ppconfig_file);
+
+FILE * fpppc = fopen (ppconfig_file, "a");
+if (fpppc)
+ {
+ int len = Ustrlen(s);
+
+ if (s[len-1] == ':' && !Ustrchr(s, ' ')) /* router, transport, ... */
+ fprintf (fpppc, "\n%s\n", s);
+ else if (strncmpic(s, US"begin ", 6) == 0) /* sections */
+ fprintf (fpppc, "\n################################################################################\n%s\n\n", s);
+ else
+ {
+ /* Count up newlines */
+ int count = 0;
+ uschar *p;
+ for (p=s; *p; ++p)
+ if (*p == '\n')
+ ++count;
+
+ /* If there are newlines, expand them into a copy buffer */
+ if (count)
+ {
+ uschar *o, *p;
+ uschar *ss = store_get(len + count*3 + 1);
+ for (p=s,o=ss; *p; ++p, ++o)
+ if (*p == '\n')
+ {
+ *o = '\\';
+ *++o = '\\';
+ *++o = '\n';
+ }
+ else
+ *o = *p;
+ fprintf (fpppc, "%s\n", ss);
+ }
+ else
+ fprintf (fpppc, "%s\n", s);
+ }
+ fclose (fpppc);
+ }
+else
+ log_write(0, LOG_CONFIG_IN, "failed to open '%s' as preprocessed config dump file",
+ ppconfig_file);
+}
+
+
/*************************************************
* Read configuration line *
*************************************************/
@@ -958,6 +1018,7 @@
if (strncmpic(s, US"begin ", 6) == 0)
{
+ dump_preprocessed_config (s);
s += 6;
while (isspace(*s)) s++;
if (big_buffer + len - s > sizeof(next_section) - 2)
@@ -967,6 +1028,8 @@
}
}
+dump_preprocessed_config (s);
+
/* Return the first non-blank character. */
return s;
Index: exim-4.52/src/readconf.c
===================================================================
--- exim-4.52.orig/src/readconf.c 2005-09-14 18:15:07.%N +0100
+++ exim-4.52/src/readconf.c 2005-09-14 18:20:42.%N +0100
@@ -1,5 +1,6 @@
/* $Cambridge: exim/exim-src/src/readconf.c,v 1.11 2005/06/27 14:29:43 ph10 Exp $ */
+#define STDERR 2
/*************************************************
* Exim - an Internet mail transport agent *
*************************************************/
@@ -588,6 +589,8 @@
{
int len = Ustrlen(s);
+ dprintf (STDERR, " ##### %s\n", s);
+
if (s[len-1] == ':' && !Ustrchr(s, ' ')) /* router, transport, ... */
fprintf (fpppc, "\n%s\n", s);
else if (strncmpic(s, US"begin ", 6) == 0) /* sections */
@@ -664,6 +667,7 @@
/* Delayed reclamation of multi-line macro resources */
if (expanding_multi_line_macro_index == -1)
{
+ dprintf (STDERR, "mlm: delayed resource reclamation: 0x%x\n", expanding_multi_line_macro);
expanding_multi_line_macro_index = 0;
expanding_multi_line_macro = NULL;
}
@@ -684,6 +688,8 @@
ss = p;
+ dprintf (STDERR, "mlm: regurgitating '%s'\n", ss);
+
/* Set index to -1 to flag delayed resource reclamation */
if (*(p + Ustrlen(p) + 2) == 0)
expanding_multi_line_macro_index = -1;
@@ -780,6 +786,7 @@
int namelen = Ustrlen(m->name);
int replen = Ustrlen(m->replacement);
+ dprintf (STDERR, "macros: found '%s'\n",m->name);
/* Expand the buffer if necessary */
while (newlen - namelen + replen + 1 > big_buffer_size)
@@ -814,6 +821,7 @@
{
expand_again = TRUE;
multi_line_macro_found = TRUE;
+ dprintf (STDERR, "mlm: expanding '%s' (%c)\n",m->name, multi_line_macro_found?'t':'f');
}
}
}
@@ -829,6 +837,7 @@
expanding_multi_line_macro = store_get (newlen + 2);
Ustrncpy (expanding_multi_line_macro, ss, newlen);
expanding_multi_line_macro_index = 0;
+ dprintf (STDERR, "mlm: stashing:\n>%s<\n", expanding_multi_line_macro);
/* Substitute \0 for \n */
for (p = expanding_multi_line_macro; *p; ++p)
@@ -837,6 +846,7 @@
/* Indicate end of lines with extra \0 */
*(p+1) = 0;
+ dprintf (STDERR, "mlm: recursing get_config_line()\n");
return get_config_line();
}
}
@@ -990,6 +1000,7 @@
{
if (big_buffer[len-1] != '\\')
{
+ dprintf (STDERR, "mlm: parsing '%s' DONE\n", s);
parsing_multi_line_macro = FALSE;
break; /* End of logical line */
}
@@ -998,6 +1009,7 @@
if (big_buffer[len-1] == '\\') /* Multi-line macro */
{
parsing_multi_line_macro = TRUE;
+ dprintf (STDERR, "mlm: parsing '%s'\n", s);
big_buffer[len-1] = '\n';
}
}
