Philip Hazel wrote:
> On Thu, 19 Sep 2002, Kevin P. Fleming wrote:
>
>
>>I've got callout verification caching code writting and (nearly)
>>working. As it turned out, it was quite easy to do, owing to how well
>>organized and understandable the Exim sources are. (I'm working with
>>Exim 4.10, of course).
>
>
> Excellent. It is always (well, usually :-) less work for me to
> incorporate a patch than to do something like this from scratch.
>
>
>>After that, it attempts to write additional records when needed, but
>>they never show up in the database file. dbfn_write is returning a
>>result of 13, but I can't figure what that's supposed to mean.
>
>
> What DBM library are you using? I'm not really an expert on any of them,
> but I would guess that this might depend on which one it is.
I'm using BDB 4.0. It works fine for all the other Exim hints databases,
though.
>
>
>>If you're out there Philip, give me a pointer as to what you think might
>>be going on. When this issue is resolved, I'll post a patch to the list
>>for others to try out.
>
>
> I'm here today, but away most of next week. If it isn't resolved by
> then, you'll have to send me the patch to try, but at the moment I'm
> giving 99.9% of my attention to getting the book produced (estimated to
> take the next month).
>
> But I'm just about to go to the dentist. I'll take a quick look at the
> source when I subsequently get in to work.
The patch is not terribly long, so I've put it on the end of this
message. It still contains one or two debug/test bits that will be
removed later, and I haven't finished updating the fixdb/dbutil code for
the new database (although dumpdb does work).
>
>
>>The feature list is:
>>- all callout verification results in records being added to the hint
>>database, unless the callout defers
>>- records are marked as positive or negative
>>- two new config options, "callout_positive_expire" (default 24 hours)
>>and "callout_negative_expire" (default 2 hours) control how long records
>>are kept in the database
>
>
> That's exactly the spec I had in mind.
>
Good... nice to know I'm on the right track.
--------
diff -X /lfs/dontdiff -urN exim-4.10/src/dbstuff.h exim-new/src/dbstuff.h
--- exim-4.10/src/dbstuff.h 2002-07-22 01:59:48.000000000 -0700
+++ exim-new/src/dbstuff.h 2002-09-19 21:13:18.000000000 -0700
@@ -540,6 +540,15 @@
uschar text[1]; /* Text message for last failure */
} dbdata_retry;
+/* This structure keeps track of addresses that have had callout
verification
+performed on them. */
+
+typedef struct {
+ time_t time_stamp;
+ /*************/
+ BOOL positive; /* Verification was positive */
+} dbdata_callout_cache;
+
/* This structure keeps track of messages that are waiting for a
particular
host for a particular transport. */
diff -X /lfs/dontdiff -urN exim-4.10/src/exim_dbutil.c
exim-new/src/exim_dbutil.c
--- exim-4.10/src/exim_dbutil.c 2002-07-22 01:59:48.000000000 -0700
+++ exim-new/src/exim_dbutil.c 2002-09-19 20:44:29.000000000 -0700
@@ -19,6 +19,7 @@
retry: retry delivery information
misc: miscellaneous hints data
wait-<t>: message waiting information; <t> is a transport name
+ callout: callout verification cache
There are a number of common subroutines, followed by three main programs,
whose inclusion is controlled by -D on the compilation command. */
@@ -62,7 +63,7 @@
#define type_retry 1
#define type_wait 2
#define type_misc 3
-
+#define type_callout 4
@@ -89,7 +90,7 @@
usage(uschar *name, uschar *options)
{
printf("Usage: exim_%s%s <spool-directory> <database-name>\n", name,
options);
-printf(" <database-name> = retry | misc | wait-<transport-name>\n");
+printf(" <database-name> = retry | misc | wait-<transport-name> |
callout\n");
exit(1);
}
@@ -110,6 +111,7 @@
if (Ustrcmp(argv[2], "retry") == 0) return type_retry;
if (Ustrcmp(argv[2], "misc") == 0) return type_misc;
if (Ustrncmp(argv[2], "wait-", 5) == 0) return type_wait;
+ if (Ustrcmp(argv[2], "callout") == 0) return type_callout;
}
usage(name, options);
return -1; /* Never obeyed */
@@ -492,6 +494,7 @@
{
dbdata_retry *retry;
dbdata_wait *wait;
+ dbdata_callout_cache *callout;
int count_bad = 0;
int i;
uschar *t;
@@ -573,6 +576,12 @@
printf("%s %s\n", print_time(((dbdata_generic *)value)->time_stamp),
keybuffer);
break;
+
+ case type_callout:
+ callout = (dbdata_callout_cache *)value;
+ printf("%s %s %s\n", print_time(((dbdata_generic
*)value)->time_stamp),
+ keybuffer, callout->positive ? "positive" : "negative");
+ break;
}
store_reset(value);
}
diff -X /lfs/dontdiff -urN exim-4.10/src/globals.c exim-new/src/globals.c
--- exim-4.10/src/globals.c 2002-07-22 01:59:48.000000000 -0700
+++ exim-new/src/globals.c 2002-09-19 19:47:40.000000000 -0700
@@ -266,6 +266,8 @@
uschar *bounce_sender_authentication = NULL;
int bsmtp_transaction_linecount = 0;
+int callout_cache_positive_expire = 24*60*60;
+int callout_cache_negative_expire = 2*60*60;
uschar *check_dns_names_pattern=
US"(?i)^(?>(?(1)\\.|())[^\\W_](?>[a-z0-9-]*[^\\W_])?)+$";
int check_log_inodes = 0;
int check_log_space = 0;
diff -X /lfs/dontdiff -urN exim-4.10/src/globals.h exim-new/src/globals.h
--- exim-4.10/src/globals.h 2002-07-22 01:59:49.000000000 -0700
+++ exim-new/src/globals.h 2002-09-19 19:47:40.000000000 -0700
@@ -134,6 +134,8 @@
extern uschar *bounce_sender_authentication; /* AUTH address for
bounces */
extern int bsmtp_transaction_linecount; /* Start of last
transaction */
+extern int callout_cache_positive_expire; /* Time for positive
callout cache records to expire */
+extern int callout_cache_negative_expire; /* Time for negative
callout cache records to expire */
extern uschar *check_dns_names_pattern;/* Regex for syntax check */
extern int check_log_inodes; /* Minimum for message
acceptance */
extern int check_log_space; /* Minimum for message
acceptance */
diff -X /lfs/dontdiff -urN exim-4.10/src/readconf.c exim-new/src/readconf.c
--- exim-4.10/src/readconf.c 2002-07-22 01:59:50.000000000 -0700
+++ exim-new/src/readconf.c 2002-09-19 19:47:40.000000000 -0700
@@ -55,6 +55,8 @@
{ "bounce_message_text", opt_stringptr, &bounce_message_text },
{ "bounce_return_message", opt_bool, &bounce_return_message },
{
"bounce_sender_authentication",opt_stringptr,&bounce_sender_authentication
},
+ { "callout_positive_expire", opt_time,
&callout_cache_positive_expire },
+ { "callout_negative_expire", opt_time,
&callout_cache_negative_expire },
{ "check_log_inodes", opt_int, &check_log_inodes },
{ "check_log_space", opt_int, &check_log_space },
{ "check_spool_inodes", opt_int, &check_spool_inodes },
diff -X /lfs/dontdiff -urN exim-4.10/src/verify.c exim-new/src/verify.c
--- exim-4.10/src/verify.c 2002-07-22 01:59:52.000000000 -0700
+++ exim-new/src/verify.c 2002-09-19 21:13:05.000000000 -0700
@@ -78,6 +78,10 @@
address_item *addr_succeed = NULL;
uschar *ko_prefix, *cr;
uschar *address = addr->orig;
+open_db dbblock;
+open_db *dbm_file;
+dbdata_callout_cache *cache_record;
+dbdata_callout_cache new_record;
/* Set up a prefix and suffix for error message which allow us to use
the same
output statements both in EXPN mode (where an SMTP response is needed)
and when
@@ -210,12 +214,54 @@
if (host_checking)
{
HDEBUG(D_verify) debug_printf("... callout omitted when host
testing\n");
- done = TRUE;
+ /* done = TRUE; */
}
- else for (host = addr->host_list;
- host != NULL && ! done;
- host = host->next)
+ /* Open the callout cache database, creating it if there isn't one.
Get any
+ cache record for this address, and if it has expired, remove it
from the
+ cache. */
+
+ cache_record = NULL;
+ dbm_file = NULL;
+ if ((dbm_file = dbfn_open(US"callout", O_RDONLY, &dbblock, FALSE))
== NULL)
+ {
+ DEBUG(D_verify)
+ debug_printf("no callout cache available\n");
+ }
+ if (dbm_file != NULL)
+ {
+ DEBUG(D_verify)
+ debug_printf("callout cache opened\n");
+ cache_record = dbfn_read(dbm_file, addr->rcpt);
+ if (cache_record != NULL)
+ {
+ DEBUG(D_verify)
+ debug_printf("got callout cache record\n");
+ if ((time(NULL) - cache_record->time_stamp) >
+ (cache_record->positive ? callout_cache_positive_expire :
callout_cache_negative_expire))
+ {
+ if (dbfn_delete(dbm_file, addr->rcpt) != 0)
+ {
+ DEBUG(D_verify)
+ debug_printf("could not remove callout cache record\n");
+ }
+ cache_record = NULL;
+ }
+ }
+ }
+
+ if (cache_record != NULL)
+ {
+ if (!cache_record->positive)
+ rc = FAIL;
+ done = TRUE;
+ DEBUG(D_verify)
+ debug_printf("used callout cache %s record\n",
cache_record->positive ? "positive" : "negative");
+ }
+
+ for (host = addr->host_list;
+ host != NULL && ! done;
+ host = host->next)
{
smtp_inblock inblock;
smtp_outblock outblock;
@@ -345,6 +391,20 @@
close(inblock.sock);
} /* Loop through all hosts, while !done */
+ /* Cache the callout result, if it was not a deferral, and if there was
+ no previous cache entry for this address. */
+ if (done && (cache_record == NULL) && (dbm_file != NULL))
+ {
+ int result;
+
+ new_record.positive = (rc != FAIL);
+ result = dbfn_write(dbm_file, addr->rcpt, &new_record,
(int)sizeof(dbdata_callout_cache));
+ DEBUG(D_verify)
+ debug_printf("dbfn_write result: %d\n", result);
+ DEBUG(D_verify)
+ debug_printf("wrote %s callout cache record\n", new_record.positive
? "positive" : "negative");
+ }
+
/* Failure to connect to any host, or any response other than 2xx
or 5xx
is a temporary error. If there was only one host, and a response was
received, leave it alone. Otherwise, give a generic response. */
@@ -358,6 +418,9 @@
is_recipient? "recipient" : "sender");
addr->basic_errno = ERRNO_CALLOUTDEFER;
}
+ if (dbm_file != NULL)
+ dbfn_close(dbm_file);
+
} /* Callout */
/* A router may return REROUTE if it has changed the value of
addr->domain
