Gitweb:
https://git.exim.org/exim.git/commitdiff/ff9663026d1a318d385730c4a2c3e85508b4b00b
Commit: ff9663026d1a318d385730c4a2c3e85508b4b00b
Parent: 5bf8a51681e171328e72f5d5b5ef8fd8a67d5f05
Author: Jeremy Harris <jgh146exb@???>
AuthorDate: Tue Feb 18 11:30:57 2020 +0000
Committer: Jeremy Harris <jgh146exb@???>
CommitDate: Tue Feb 18 11:30:57 2020 +0000
Overlapped twophase-queue-run and delivery. Experimental.
---
doc/doc-txt/experimental-spec.txt | 24 ++
src/src/EDITME | 3 +
src/src/child.c | 3 +-
src/src/config.h.defaults | 1 +
src/src/daemon.c | 167 +++++++-
src/src/deliver.c | 2 +
src/src/exim.c | 26 +-
src/src/functions.h | 7 +-
src/src/globals.c | 7 +
src/src/globals.h | 7 +
src/src/macros.h | 5 +
src/src/queue.c | 35 +-
src/src/readconf.c | 3 +
src/src/smtp_out.c | 3 +-
src/src/spool_in.c | 6 +-
src/src/transport.c | 7 +-
test/confs/0999 | 56 +++
test/scripts/0999-EXP-Queue-Ramp/0999 | 684 ++++++++++++++++++++++++++++++
test/scripts/0999-EXP-Queue-Ramp/REQUIRES | 1 +
test/stderr/0999 | 6 +
20 files changed, 1026 insertions(+), 27 deletions(-)
diff --git a/doc/doc-txt/experimental-spec.txt b/doc/doc-txt/experimental-spec.txt
index 5b18b7b..3beab4b 100644
--- a/doc/doc-txt/experimental-spec.txt
+++ b/doc/doc-txt/experimental-spec.txt
@@ -837,6 +837,30 @@ and a whitespace-separated port number must be given.
+Twophase queue run fast ramp
+----------------------------
+To include this feature, add to Local/Makefile:
+ EXPERIMENTAL_QUEUE_RAMP=yes
+
+If the (added for this feature) main-section option "queue_fast_ramp" (boolean)
+is set, and a two-phase ("-qq") queue run finds, during the first phase, a
+suitably large number of message routed for a given host - then (subject to
+the usual queue-runner resource limits) delivery for that host is initiated
+immediately, overlapping with the remainder of the first phase.
+
+This is incompatible with queue_run_in_order.
+
+The result should be a faster startup of deliveries when a large queue is
+present and reasonable numbers of messages are routed to common hosts; this
+could be a smarthost case, or delivery onto the Internet where a large proportion
+of recipients hapen to be on a Gorilla-sized provider.
+
+As usual, the presence of a configuration option is associated with a
+predefined macro, making it possible to write portable configurations.
+For this one, the macro is _OPT_MAIN_QUEUE_FAST_RAMP.
+
+
+
--------------------------------------------------------------
End of file
--------------------------------------------------------------
diff --git a/src/src/EDITME b/src/src/EDITME
index 352bc7d..8d85523 100644
--- a/src/src/EDITME
+++ b/src/src/EDITME
@@ -632,6 +632,9 @@ DISABLE_MAL_MKS=yes
# Uncomment the following line to include support for TLS Resumption
# EXPERIMENTAL_TLS_RESUME=yes
+# Uncomment the following to include the fast-ramp two-phase-queue-run support
+# EXPERIMENTAL_QUEUE_RAMP=yes
+
###############################################################################
# THESE ARE THINGS YOU MIGHT WANT TO SPECIFY #
###############################################################################
diff --git a/src/src/child.c b/src/src/child.c
index d3cd882..c5054b6 100644
--- a/src/src/child.c
+++ b/src/src/child.c
@@ -75,7 +75,7 @@ int n = 0;
int extra = pcount ? *pcount : 0;
uschar **argv;
-argv = store_get((extra + acount + MAX_CLMACROS + 18) * sizeof(char *), FALSE);
+argv = store_get((extra + acount + MAX_CLMACROS + 19) * sizeof(char *), FALSE);
/* In all case, the list starts out with the path, any macros, and a changed
config file. */
@@ -109,6 +109,7 @@ if (!minimal)
if (debug_selector != 0)
argv[n++] = string_sprintf("-d=0x%x", debug_selector);
}
+ if (!f.testsuite_delays) argv[n++] = US"-odd";
if (f.dont_deliver) argv[n++] = US"-N";
if (f.queue_smtp) argv[n++] = US"-odqs";
if (f.synchronous_delivery) argv[n++] = US"-odi";
diff --git a/src/src/config.h.defaults b/src/src/config.h.defaults
index 223e2d6..9d77f30 100644
--- a/src/src/config.h.defaults
+++ b/src/src/config.h.defaults
@@ -202,6 +202,7 @@ Do not put spaces between # and the 'define'.
#define EXPERIMENTAL_DCC
#define EXPERIMENTAL_DSN_INFO
#define EXPERIMENTAL_LMDB
+#define EXPERIMENTAL_QUEUE_RAMP
#define EXPERIMENTAL_QUEUEFILE
#define EXPERIMENTAL_SRS
#define EXPERIMENTAL_SRS_NATIVE
diff --git a/src/src/daemon.c b/src/src/daemon.c
index ddfd8e7..aedd3fb 100644
--- a/src/src/daemon.c
+++ b/src/src/daemon.c
@@ -973,6 +973,102 @@ exim_exit(EXIT_SUCCESS, US"daemon");
}
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+/*************************************************
+* Listener socket for local work prompts *
+*************************************************/
+
+static void
+daemon_notifier_socket(void)
+{
+int fd;
+const uschar * where;
+struct sockaddr_un sun = {.sun_family = AF_UNIX};
+
+DEBUG(D_any) debug_printf("creating notifier socket\n");
+
+where = US"socket";
+#ifdef SOCK_CLOEXEC
+if ((fd = socket(AF_UNIX, SOCK_DGRAM|SOCK_CLOEXEC, 0)) < 0)
+ goto bad;
+#else
+if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0))) < 0)
+ goto bad;
+(void)fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC);
+#endif
+
+snprintf(sun.sun_path, sizeof(sun.sun_path), "%s/%s",
+ spool_directory, NOTIFIER_SOCKET_NAME);
+where = US"bind";
+if (bind(fd, (const struct sockaddr *)&sun, sizeof(sun)) < 0)
+ goto bad;
+
+where = US"SO_PASSCRED";
+if (setsockopt(fd, SOL_SOCKET, SO_PASSCRED, &on, sizeof(on)) < 0)
+ goto bad;
+
+/* debug_printf("%s: fd %d\n", __FUNCTION__, fd); */
+daemon_notifier_fd = fd;
+return;
+
+bad:
+ log_write(0, LOG_MAIN|LOG_PANIC, "%s: %s: %s",
+ __FUNCTION__, where, strerror(errno));
+}
+
+
+static uschar queuerun_msgid[MESSAGE_ID_LENGTH+1];
+
+/* Return TRUE if a sigalrm should be emulated */
+static BOOL
+daemon_notification(void)
+{
+uschar buf[256], cbuf[256];
+struct iovec iov = {.iov_base = buf, .iov_len = sizeof(buf)-1};
+struct msghdr msg = { .msg_name = NULL,
+ .msg_namelen = 0,
+ .msg_iov = &iov,
+ .msg_iovlen = 1,
+ .msg_control = cbuf,
+ .msg_controllen = sizeof(cbuf)
+ };
+ssize_t sz;
+struct cmsghdr * cp;
+
+buf[sizeof(buf)-1] = 0;
+if ((sz = recvmsg(daemon_notifier_fd, &msg, 0)) <= 0) return FALSE;
+if (sz >= sizeof(buf)) return FALSE;
+
+for (struct cmsghdr * cp = CMSG_FIRSTHDR(&msg);
+ cp;
+ cp = CMSG_NXTHDR(&msg, cp))
+ if (cp->cmsg_level == SOL_SOCKET && cp->cmsg_type == SCM_CREDENTIALS)
+ {
+ struct ucred * cr = (struct ucred *) CMSG_DATA(cp);
+ if (cr->uid && cr->uid != exim_uid)
+ {
+ DEBUG(D_queue_run) debug_printf("%s: sender creds pid %d uid %d gid %d\n",
+ __FUNCTION__, (int)cr->pid, (int)cr->uid, (int)cr->gid);
+ return FALSE;
+ }
+ break;
+ }
+
+buf[sz] = 0;
+switch (buf[0])
+ {
+ case NOTIFY_MSG_QRUN:
+ /* this should be a message_id */
+ DEBUG(D_queue_run)
+ debug_printf("%s: qrunner trigger: %s\n", __FUNCTION__, buf+1);
+ memcpy(queuerun_msgid, buf+1, MESSAGE_ID_LENGTH+1);
+ return TRUE;
+ }
+return FALSE;
+}
+#endif /*EXPERIMENTAL_QUEUE_RAMP*/
+
+
/*************************************************
* Exim Daemon Mainline *
*************************************************/
@@ -1418,6 +1514,11 @@ if (f.background_daemon)
/* We are now in the disconnected, daemon process (unless debugging). Set up
the listening sockets if required. */
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+if (queue_fast_ramp)
+ daemon_notifier_socket();
+#endif
+
if (f.daemon_listen && !f.inetd_wait_mode)
{
int sk;
@@ -1693,7 +1794,7 @@ if (f.inetd_wait_mode)
set_process_info("daemon(%s): pre-listening socket", version_string);
/* set up the timeout logic */
- sigalrm_seen = 1;
+ sigalrm_seen = TRUE;
}
else if (f.daemon_listen)
@@ -1921,7 +2022,11 @@ for (;;)
else
{
- DEBUG(D_any) debug_printf("SIGALRM received\n");
+ DEBUG(D_any) debug_printf("%s received\n",
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ *queuerun_msgid ? "qrun notification" :
+#endif
+ "SIGALRM");
/* Do a full queue run in a child process, if required, unless we already
have enough queue runners on the go. If we are not running as root, a
@@ -1943,8 +2048,12 @@ for (;;)
/* Close any open listening sockets in the child */
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ if (daemon_notifier_fd >= 0)
+ (void) close(daemon_notifier_fd);
+#endif
for (int sk = 0; sk < listen_socket_count; sk++)
- (void)close(listen_sockets[sk]);
+ (void) close(listen_sockets[sk]);
/* Reset SIGHUP and SIGCHLD in the child in both cases. */
@@ -1959,13 +2068,17 @@ for (;;)
{
uschar opt[8];
uschar *p = opt;
- uschar *extra[5];
+ uschar *extra[7];
int extracount = 1;
signal(SIGALRM, SIG_DFL);
*p++ = '-';
*p++ = 'q';
- if (f.queue_2stage) *p++ = 'q';
+ if ( f.queue_2stage
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ && !*queuerun_msgid
+#endif
+ ) *p++ = 'q';
if (f.queue_run_first_delivery) *p++ = 'i';
if (f.queue_run_force) *p++ = 'f';
if (f.deliver_force_thaw) *p++ = 'f';
@@ -1974,6 +2087,14 @@ for (;;)
extra[0] = *queue_name
? string_sprintf("%sG%s", opt, queue_name) : opt;
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ if (*queuerun_msgid)
+ {
+ extra[extracount++] = queuerun_msgid; /* Trigger only the */
+ extra[extracount++] = queuerun_msgid; /* one message */
+ }
+#endif
+
/* If -R or -S were on the original command line, ensure they get
passed on. */
@@ -1992,15 +2113,23 @@ for (;;)
/* Overlay this process with a new execution. */
- (void)child_exec_exim(CEE_EXEC_PANIC, FALSE, NULL, TRUE, extracount,
- extra[0], extra[1], extra[2], extra[3], extra[4]);
+ (void)child_exec_exim(CEE_EXEC_PANIC, FALSE, NULL, FALSE, extracount,
+ extra[0], extra[1], extra[2], extra[3], extra[4], extra[5], extra[6]);
/* Control never returns here. */
}
/* No need to re-exec; SIGALRM remains set to the default handler */
- queue_run(NULL, NULL, FALSE);
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ if (*queuerun_msgid)
+ {
+ f.queue_2stage = FALSE;
+ queue_run(queuerun_msgid, queuerun_msgid, FALSE);
+ }
+ else
+#endif
+ queue_run(NULL, NULL, FALSE);
exim_underbar_exit(EXIT_SUCCESS);
}
@@ -2027,7 +2156,12 @@ for (;;)
/* Reset the alarm clock */
sigalrm_seen = FALSE;
- ALARM(queue_interval);
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ if (*queuerun_msgid)
+ *queuerun_msgid = 0;
+ else
+#endif
+ ALARM(queue_interval);
}
} /* sigalrm_seen */
@@ -2050,6 +2184,10 @@ for (;;)
fd_set select_listen;
FD_ZERO(&select_listen);
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ if (daemon_notifier_fd >= 0)
+ FD_SET(daemon_notifier_fd, &select_listen);
+#endif
for (int sk = 0; sk < listen_socket_count; sk++)
{
FD_SET(listen_sockets[sk], &select_listen);
@@ -2105,6 +2243,16 @@ for (;;)
int accept_socket = -1;
if (!select_failed)
+ {
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ if ( daemon_notifier_fd >= 0
+ && FD_ISSET(daemon_notifier_fd, &select_listen))
+ {
+ FD_CLR(daemon_notifier_fd, &select_listen);
+ sigalrm_seen = daemon_notification();
+ break; /* to top of daemon loop */
+ }
+#endif
for (int sk = 0; sk < listen_socket_count; sk++)
if (FD_ISSET(listen_sockets[sk], &select_listen))
{
@@ -2114,6 +2262,7 @@ for (;;)
FD_CLR(listen_sockets[sk], &select_listen);
break;
}
+ }
/* If select or accept has failed and this was not caused by an
interruption, log the incident and try again. With asymmetric TCP/IP
diff --git a/src/src/deliver.c b/src/src/deliver.c
index c4160a5..4678138 100644
--- a/src/src/deliver.c
+++ b/src/src/deliver.c
@@ -4642,6 +4642,7 @@ all pipes, so I do not see a reason to use non-blocking IO here
search_tidyup();
+ DEBUG(D_deliver) debug_printf("forking transport process\n");
if ((pid = fork()) == 0)
{
int fd = pfd[pipe_write];
@@ -4972,6 +4973,7 @@ all pipes, so I do not see a reason to use non-blocking IO here
(void)close(fd);
exit(EXIT_SUCCESS);
}
+ DEBUG(D_deliver) debug_printf("forked transport process (%d)\n", pid);
/* Back in the mainline: close the unwanted half of the pipe. */
diff --git a/src/src/exim.c b/src/src/exim.c
index 98174d6..a8f3c22 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -972,15 +972,6 @@ fprintf(fp, "Support for:");
tcp_init();
if (f.tcp_fastopen_ok) fprintf(fp, " TCP_Fast_Open");
#endif
-#ifdef EXPERIMENTAL_LMDB
- fprintf(fp, " Experimental_LMDB");
-#endif
-#ifdef EXPERIMENTAL_QUEUEFILE
- fprintf(fp, " Experimental_QUEUEFILE");
-#endif
-#if defined(EXPERIMENTAL_SRS) || defined(EXPERIMENTAL_SRS_NATIVE)
- fprintf(fp, " Experimental_SRS");
-#endif
#ifdef EXPERIMENTAL_ARC
fprintf(fp, " Experimental_ARC");
#endif
@@ -993,6 +984,18 @@ fprintf(fp, "Support for:");
#ifdef EXPERIMENTAL_DSN_INFO
fprintf(fp, " Experimental_DSN_info");
#endif
+#ifdef EXPERIMENTAL_LMDB
+ fprintf(fp, " Experimental_LMDB");
+#endif
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ fprintf(fp, " Experimental_Queue_Ramp");
+#endif
+#ifdef EXPERIMENTAL_QUEUEFILE
+ fprintf(fp, " Experimental_QUEUEFILE");
+#endif
+#if defined(EXPERIMENTAL_SRS) || defined(EXPERIMENTAL_SRS_NATIVE)
+ fprintf(fp, " Experimental_SRS");
+#endif
#ifdef EXPERIMENTAL_TLS_RESUME
fprintf(fp, " Experimental_TLS_resume");
#endif
@@ -3006,6 +3009,11 @@ for (i = 1; i < argc; i++)
queue_only_set = TRUE;
}
+ /* -odd: testsuite-only: add no inter-process delays */
+
+ else if (Ustrcmp(argrest, "d") == 0)
+ f.testsuite_delays = FALSE;
+
/* -odf: foreground delivery (smail-compatible option); same effect as
-odi: interactive (synchronous) delivery (sendmail-compatible option)
*/
diff --git a/src/src/functions.h b/src/src/functions.h
index 8b04d58..9716a02 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -363,8 +363,11 @@ extern int vaguely_random_number_fallback(int);
extern BOOL queue_action(uschar *, int, uschar **, int, int);
extern void queue_check_only(void);
-extern void queue_list(int, uschar **, int);
extern void queue_count(void);
+extern void queue_list(int, uschar **, int);
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+extern void queue_notify_daemon(const uschar * hostname);
+#endif
extern void queue_run(uschar *, uschar *, BOOL);
extern int random_number(int);
@@ -1043,7 +1046,7 @@ static inline void
testharness_pause_ms(int millisec)
{
#ifndef MEASURE_TIMING
-if (f.running_in_test_harness) millisleep(millisec);
+if (f.running_in_test_harness && f.testsuite_delays) millisleep(millisec);
#endif
}
diff --git a/src/src/globals.c b/src/src/globals.c
index 53a4d12..458ab48 100644
--- a/src/src/globals.c
+++ b/src/src/globals.c
@@ -313,6 +313,7 @@ struct global_flags f =
.system_filtering = FALSE,
.taint_check_slow = FALSE,
+ .testsuite_delays = TRUE,
.tcp_fastopen_ok = FALSE,
.tcp_in_fastopen = FALSE,
.tcp_in_fastopen_data = FALSE,
@@ -379,6 +380,9 @@ BOOL prod_requires_admin = TRUE;
BOOL proxy_session = FALSE;
#endif
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+BOOL queue_fast_ramp = FALSE;
+#endif
BOOL queue_list_requires_admin = TRUE;
BOOL queue_only = FALSE;
BOOL queue_only_load_latch = TRUE;
@@ -736,6 +740,9 @@ cut_t cutthrough = {
.nrcpt = 0, /* number of addresses */
};
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+int daemon_notifier_fd = -1;
+#endif
uschar *daemon_smtp_port = US"smtp";
int daemon_startup_retries = 9;
int daemon_startup_sleep = 30;
diff --git a/src/src/globals.h b/src/src/globals.h
index 74af185..88751f3 100644
--- a/src/src/globals.h
+++ b/src/src/globals.h
@@ -275,6 +275,7 @@ extern struct global_flags {
BOOL system_filtering :1; /* TRUE when running system filter */
BOOL taint_check_slow :1; /* malloc/mmap are not returning distinct ranges */
+ BOOL testsuite_delays :1; /* interprocess sequencing delays, under testsuite */
BOOL tcp_fastopen_ok :1; /* appears to be supported by kernel */
BOOL tcp_in_fastopen :1; /* conn usefully used fastopen */
BOOL tcp_in_fastopen_data :1; /* fastopen carried data */
@@ -446,6 +447,9 @@ typedef struct {
} cut_t;
extern cut_t cutthrough; /* Deliver-concurrently */
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+extern int daemon_notifier_fd; /* Unix socket for notifications */
+#endif
extern uschar *daemon_smtp_port; /* Can be a list of ports */
extern int daemon_startup_retries; /* Number of times to retry */
extern int daemon_startup_sleep; /* Sleep between retries */
@@ -786,6 +790,9 @@ extern uschar *prvscheck_result; /* Set during prvscheck expansion item */
extern const uschar *qualify_domain_recipient; /* Domain to qualify recipients with */
extern uschar *qualify_domain_sender; /* Domain to qualify senders with */
extern uschar *queue_domains; /* Queue these domains */
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+extern BOOL queue_fast_ramp; /* 2-phase queue-run overlap */
+#endif
extern BOOL queue_list_requires_admin; /* TRUE if -bp requires admin */
/* immediate children */
extern pid_t queue_run_pid; /* PID of the queue running process or 0 */
diff --git a/src/src/macros.h b/src/src/macros.h
index c99b152..ca61f53 100644
--- a/src/src/macros.h
+++ b/src/src/macros.h
@@ -1100,4 +1100,9 @@ should not be one active. */
#define SVFMT_TAINT_NOCHK BIT(2)
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+# define NOTIFIER_SOCKET_NAME "exim_daemon_notify"
+# define NOTIFY_MSG_QRUN 1 /* Notify message types */
+#endif
+
/* End of macros.h */
diff --git a/src/src/queue.c b/src/src/queue.c
index d472b98..3c72ead 100644
--- a/src/src/queue.c
+++ b/src/src/queue.c
@@ -346,7 +346,7 @@ const pcre *selectstring_regex_sender = NULL;
uschar *log_detail = NULL;
int subcount = 0;
uschar subdirs[64];
-pid_t qpid[4] = {0}; /* Parallelism factor for q2stage 1st phase */
+pid_t qpid[1] = {0}; /* Parallelism factor for q2stage 1st phase */
#ifdef MEASURE_TIMING
report_time_since(×tamp_startup, US"queue_run start");
@@ -1491,6 +1491,39 @@ if (s)
}
}
+
+
+/******************************************************************************/
+/******************************************************************************/
+
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+void
+queue_notify_daemon(const uschar * msgid)
+{
+uschar buf[MESSAGE_ID_LENGTH + 2];
+int fd;
+
+DEBUG(D_queue_run) debug_printf("%s: %s\n", __FUNCTION__, msgid);
+
+buf[0] = NOTIFY_MSG_QRUN;
+memcpy(buf+1, msgid, MESSAGE_ID_LENGTH+1);
+
+if ((fd = socket(AF_UNIX, SOCK_DGRAM, 0)) >= 0)
+ {
+ struct sockaddr_un sun = {.sun_family = AF_UNIX};
+
+ snprintf(sun.sun_path, sizeof(sun.sun_path), "%s/%s",
+ spool_directory, NOTIFIER_SOCKET_NAME);
+
+ if (sendto(fd, buf, sizeof(buf), 0, &sun, sizeof(sun)) < 0)
+ DEBUG(D_queue_run)
+ debug_printf("%s: sendto %s\n", __FUNCTION__, strerror(errno));
+ close(fd);
+ }
+else DEBUG(D_queue_run) debug_printf(" socket: %s\n", strerror(errno));
+}
+#endif
+
#endif /*!COMPILE_UTILITY*/
/* End of queue.c */
diff --git a/src/src/readconf.c b/src/src/readconf.c
index f16f51d..c8a3dff 100644
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -259,6 +259,9 @@ static optionlist optionlist_config[] = {
{ "qualify_domain", opt_stringptr, {&qualify_domain_sender} },
{ "qualify_recipient", opt_stringptr, {&qualify_domain_recipient} },
{ "queue_domains", opt_stringptr, {&queue_domains} },
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ { "queue_fast_ramp", opt_bool, {&queue_fast_ramp} },
+#endif
{ "queue_list_requires_admin",opt_bool, {&queue_list_requires_admin} },
{ "queue_only", opt_bool, {&queue_only} },
{ "queue_only_file", opt_stringptr, {&queue_only_file} },
diff --git a/src/src/smtp_out.c b/src/src/smtp_out.c
index 96ee152..12ed5bc 100644
--- a/src/src/smtp_out.c
+++ b/src/src/smtp_out.c
@@ -500,7 +500,7 @@ else
rc = n;
}
else
-
+ {
rc = send(outblock->cctx->sock, outblock->buffer, n,
#ifdef MSG_MORE
more ? MSG_MORE : 0
@@ -508,6 +508,7 @@ else
0
#endif
);
+ }
}
if (rc <= 0)
diff --git a/src/src/spool_in.c b/src/src/spool_in.c
index 575c398..5f8a822 100644
--- a/src/src/spool_in.c
+++ b/src/src/spool_in.c
@@ -105,9 +105,9 @@ lock_data.l_len = SPOOL_DATA_START_OFFSET;
if (fcntl(fd, F_SETLK, &lock_data) < 0)
{
- log_write(L_skip_delivery,
- LOG_MAIN,
- "Spool file is locked (another process is handling this message)");
+ log_write(L_skip_delivery, LOG_MAIN,
+ "Spool file for %s is locked (another process is handling this message)",
+ id);
(void)close(fd);
errno = 0;
return -1;
diff --git a/src/src/transport.c b/src/src/transport.c
index 02994d2..d9eba16 100644
--- a/src/src/transport.c
+++ b/src/src/transport.c
@@ -1560,12 +1560,17 @@ for (host_item * host = hostlist; host; host = host->next)
/* If this record is full, write it out with a new name constructed
from the sequence number, increase the sequence number, and empty
- the record. */
+ the record. If we're doing a two-phase queue run initial phase, ping the
+ daemon to consider running a delivery on this host. */
if (host_record->count >= WAIT_NAME_MAX)
{
sprintf(CS buffer, "%.200s:%d", host->name, host_record->sequence);
dbfn_write(dbm_file, buffer, host_record, sizeof(dbdata_wait) + host_length);
+#ifdef EXPERIMENTAL_QUEUE_RAMP
+ if (f.queue_2stage && queue_fast_ramp && !queue_run_in_order)
+ queue_notify_daemon(message_id);
+#endif
host_record->sequence++;
host_record->count = 0;
host_length = 0;
diff --git a/test/confs/0999 b/test/confs/0999
new file mode 100644
index 0000000..c3a2ad4
--- /dev/null
+++ b/test/confs/0999
@@ -0,0 +1,56 @@
+# Exim test configuration 0999
+# Queue many messages for a two-phase fast-ramp run
+
+hostlist loopback = <; 127.0.0.0/8 ; 0.0.0.0 ; ::1 ; 0000:0000:0000:0000:0000:ffff
+untrusted_set_sender = *
+
+SERVER =
+
+.include DIR/aux-var/std_conf_prefix
+
+rfc1413_query_timeout = 0s
+log_selector = +sender_on_delivery +millisec
+
+# ----- Main settings -----
+
+acl_smtp_rcpt = accept
+
+queue_only
+queue_fast_ramp
+
+# ----- Routers -----
+
+begin routers
+
+client:
+ driver = accept
+ condition = ${if eq {SERVER}{server}{no}{yes}}
+ transport = send_to_server
+
+server:
+ driver = accept
+ transport = send_to_server
+
+
+# ----- Transports -----
+
+begin transports
+
+send_to_server:
+ driver = smtp
+ connection_max_messages = 0
+ allow_localhost
+ hosts = 127.0.0.1
+ port = PORT_D
+ hosts_try_fastopen = :
+ # assumes that HOSTIPV4 can send to 127.0.0.1
+ interface = ${if eq {$sender_address_domain}{dustybelt.tld} {127.0.0.1}{HOSTIPV4}}
+
+# ----- Retry -----
+
+begin retry
+
+* * F,5d,10s
+
+# End
+
diff --git a/test/scripts/0999-EXP-Queue-Ramp/0999 b/test/scripts/0999-EXP-Queue-Ramp/0999
new file mode 100644
index 0000000..fd55215
--- /dev/null
+++ b/test/scripts/0999-EXP-Queue-Ramp/0999
@@ -0,0 +1,684 @@
+# fast-ramp continued-delivery queue run
+# Exim test configuration 0999
+#
+# This feature has testability problems, because it results in
+# parallel processing of the queue by two concurrent processes
+# - the daemon, having been notified by the manual "-qq" process
+# once a sufficient list for the destination has been built, and
+# the aforementioned "-qq" process once it completes the first phase.
+# We don't really want to add yet another testsuite-only option to
+# force the latter to not be done.
+# So the best we can do is check that at least some deliveries were
+# made by the daemon.
+#
+exim -DSERVER=server -bd -q30m -odd -oX PORT_D
+****
+#
+exim -bs
+mail from:ralph@???
+rcpt to:bob@???
+data
+This is a test message.
+It has three lines.
+This is the last line.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 1
+
+This is message number 1.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 2
+
+This is message number 2.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 3
+
+This is message number 3.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 4
+
+This is message number 4.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 5
+
+This is message number 5.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 6
+
+This is message number 6.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 7
+
+This is message number 7.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 8
+
+This is message number 8.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 9
+
+This is message number 9.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 10
+
+This is message number 10.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 11
+
+This is message number 11.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 12
+
+This is message number 12.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 13
+
+This is message number 13.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 14
+
+This is message number 14.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 15
+
+This is message number 15.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 16
+
+This is message number 16.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 17
+
+This is message number 17.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 18
+
+This is message number 18.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 19
+
+This is message number 19.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 20
+
+This is message number 20.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 21
+
+This is message number 21.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 22
+
+This is message number 22.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 23
+
+This is message number 23.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 24
+
+This is message number 24.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 25
+
+This is message number 25.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 26
+
+This is message number 26.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 27
+
+This is message number 27.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 28
+
+This is message number 28.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 29
+
+This is message number 29.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 30
+
+This is message number 30.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 31
+
+This is message number 31.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 32
+
+This is message number 32.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 33
+
+This is message number 33.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 34
+
+This is message number 34.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 35
+
+This is message number 35.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 36
+
+This is message number 36.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 37
+
+This is message number 37.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 38
+
+This is message number 38.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 39
+
+This is message number 39.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 40
+
+This is message number 40.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 41
+
+This is message number 41.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 42
+
+This is message number 42.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 43
+
+This is message number 43.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 44
+
+This is message number 44.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 45
+
+This is message number 45.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 46
+
+This is message number 46.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 47
+
+This is message number 47.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 48
+
+This is message number 48.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 49
+
+This is message number 49.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 50
+
+This is message number 50.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 51
+
+This is message number 51.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 52
+
+This is message number 52.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 53
+
+This is message number 53.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 54
+
+This is message number 54.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 55
+
+This is message number 55.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 56
+
+This is message number 56.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 57
+
+This is message number 57.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 58
+
+This is message number 58.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 59
+
+This is message number 59.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 60
+
+This is message number 60.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 61
+
+This is message number 61.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 62
+
+This is message number 62.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 63
+
+This is message number 63.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 64
+
+This is message number 64.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 65
+
+This is message number 65.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 66
+
+This is message number 66.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 67
+
+This is message number 67.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 68
+
+This is message number 68.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 69
+
+This is message number 69.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 70
+
+This is message number 70.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 71
+
+This is message number 71.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 72
+
+This is message number 72.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 73
+
+This is message number 73.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 74
+
+This is message number 74.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 75
+
+This is message number 75.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 76
+
+This is message number 76.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 77
+
+This is message number 77.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 78
+
+This is message number 78.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 79
+
+This is message number 79.
+.
+RSET
+mail from:ralph@???
+rcpt to:bob@???
+data
+Subject: message_id 80
+
+This is message number 80.
+.
+quit
+****
+#
+#
+exim -odd -qq
+****
+#
+#
+killdaemon
+#
+# Only check that we logged the right number of messages; don't care
+# about ordering or mistakes in wrong message-id
+sudo perl
+system "egrep -v '(Completed|<=|=>)' DIR/spool/log/mainlog 1>&2";
+system "wc -l DIR/test-stdout 1>&2";
+system "grep -q '=>' DIR/spool/log/servermainlog && echo 'daemon did make at least one delivery' 1>&2";
+****
+sudo rm DIR/spool/log/mainlog DIR/spool/log/servermainlog
+no_stdout_check
+no_msglog_check
diff --git a/test/scripts/0999-EXP-Queue-Ramp/REQUIRES b/test/scripts/0999-EXP-Queue-Ramp/REQUIRES
new file mode 100644
index 0000000..bde27c3
--- /dev/null
+++ b/test/scripts/0999-EXP-Queue-Ramp/REQUIRES
@@ -0,0 +1 @@
+support Experimental_Queue_Ramp
diff --git a/test/stderr/0999 b/test/stderr/0999
new file mode 100644
index 0000000..1b45b63
--- /dev/null
+++ b/test/stderr/0999
@@ -0,0 +1,6 @@
+2017-07-30 18:51:05.712 Start queue run: pid=pppp -qq
+2017-07-30 18:51:05.712 End queue run: pid=pppp -qq
+406 TESTSUITE/test-stdout
+daemon did make at least one delivery
+
+******** SERVER ********
