Re: [Exim] New exim code having DB writing problems

Top Page
Delete this message
Reply to this message
Author: Kevin P. Fleming
Date:  
To: exim-users
Subject: Re: [Exim] New exim code having DB writing problems
Philip Hazel wrote:
> On Thu, 19 Sep 2002, Kevin P. Fleming wrote:
>
>
>>I've got callout verification caching code writting and (nearly)
>>working. As it turned out, it was quite easy to do, owing to how well
>>organized and understandable the Exim sources are. (I'm working with
>>Exim 4.10, of course).
>
>
> Excellent. It is always (well, usually :-) less work for me to
> incorporate a patch than to do something like this from scratch.
>
>
>>After that, it attempts to write additional records when needed, but
>>they never show up in the database file. dbfn_write is returning a
>>result of 13, but I can't figure what that's supposed to mean.
>
>
> What DBM library are you using? I'm not really an expert on any of them,
> but I would guess that this might depend on which one it is.


I'm using BDB 4.0. It works fine for all the other Exim hints databases,
though.

>
>
>>If you're out there Philip, give me a pointer as to what you think might
>>be going on. When this issue is resolved, I'll post a patch to the list
>>for others to try out.
>
>
> I'm here today, but away most of next week. If it isn't resolved by
> then, you'll have to send me the patch to try, but at the moment I'm
> giving 99.9% of my attention to getting the book produced (estimated to
> take the next month).
>
> But I'm just about to go to the dentist. I'll take a quick look at the
> source when I subsequently get in to work.


The patch is not terribly long, so I've put it on the end of this
message. It still contains one or two debug/test bits that will be
removed later, and I haven't finished updating the fixdb/dbutil code for
the new database (although dumpdb does work).

>
>
>>The feature list is:
>>- all callout verification results in records being added to the hint
>>database, unless the callout defers
>>- records are marked as positive or negative
>>- two new config options, "callout_positive_expire" (default 24 hours)
>>and "callout_negative_expire" (default 2 hours) control how long records
>>are kept in the database
>
>
> That's exactly the spec I had in mind.
>


Good... nice to know I'm on the right track.

--------

diff -X /lfs/dontdiff -urN exim-4.10/src/dbstuff.h exim-new/src/dbstuff.h
--- exim-4.10/src/dbstuff.h    2002-07-22 01:59:48.000000000 -0700
+++ exim-new/src/dbstuff.h    2002-09-19 21:13:18.000000000 -0700
@@ -540,6 +540,15 @@
    uschar text[1];         /* Text message for last failure */
  } dbdata_retry;


+/* This structure keeps track of addresses that have had callout
verification
+performed on them. */
+
+typedef struct {
+  time_t time_stamp;
+  /*************/
+  BOOL   positive;        /* Verification was positive */
+} dbdata_callout_cache;
+
  /* This structure keeps track of messages that are waiting for a
particular
  host for a particular transport. */


diff -X /lfs/dontdiff -urN exim-4.10/src/exim_dbutil.c
exim-new/src/exim_dbutil.c
--- exim-4.10/src/exim_dbutil.c    2002-07-22 01:59:48.000000000 -0700
+++ exim-new/src/exim_dbutil.c    2002-09-19 20:44:29.000000000 -0700
@@ -19,6 +19,7 @@
    retry:      retry delivery information
    misc:       miscellaneous hints data
    wait-<t>:   message waiting information; <t> is a transport name
+  callout:    callout verification cache


There are a number of common subroutines, followed by three main programs,
whose inclusion is controlled by -D on the compilation command. */
@@ -62,7 +63,7 @@
#define type_retry 1
#define type_wait 2
#define type_misc 3
-
+#define type_callout 4



@@ -89,7 +90,7 @@
  usage(uschar *name, uschar *options)
  {
  printf("Usage: exim_%s%s  <spool-directory> <database-name>\n", name,
options);
-printf("       <database-name> = retry | misc | wait-<transport-name>\n");
+printf("       <database-name> = retry | misc | wait-<transport-name> |
callout\n");
  exit(1);
  }


@@ -110,6 +111,7 @@
    if (Ustrcmp(argv[2], "retry") == 0) return type_retry;
    if (Ustrcmp(argv[2], "misc") == 0) return type_misc;
    if (Ustrncmp(argv[2], "wait-", 5) == 0) return type_wait;
+  if (Ustrcmp(argv[2], "callout") == 0) return type_callout;
    }
  usage(name, options);
  return -1;              /* Never obeyed */
@@ -492,6 +494,7 @@
    {
    dbdata_retry *retry;
    dbdata_wait *wait;
+  dbdata_callout_cache *callout;
    int count_bad = 0;
    int i;
    uschar *t;
@@ -573,6 +576,12 @@
        printf("%s %s\n", print_time(((dbdata_generic *)value)->time_stamp),
          keybuffer);
        break;
+
+      case type_callout:
+      callout = (dbdata_callout_cache *)value;
+      printf("%s %s %s\n", print_time(((dbdata_generic
*)value)->time_stamp),
+        keybuffer, callout->positive ? "positive" : "negative");
+      break;
        }
      store_reset(value);
      }
diff -X /lfs/dontdiff -urN exim-4.10/src/globals.c exim-new/src/globals.c
--- exim-4.10/src/globals.c    2002-07-22 01:59:48.000000000 -0700
+++ exim-new/src/globals.c    2002-09-19 19:47:40.000000000 -0700
@@ -266,6 +266,8 @@
  uschar *bounce_sender_authentication = NULL;
  int     bsmtp_transaction_linecount = 0;


+int     callout_cache_positive_expire = 24*60*60;
+int     callout_cache_negative_expire = 2*60*60;
  uschar *check_dns_names_pattern=
US"(?i)^(?>(?(1)\\.|())[^\\W_](?>[a-z0-9-]*[^\\W_])?)+$";
  int     check_log_inodes       = 0;
  int     check_log_space        = 0;
diff -X /lfs/dontdiff -urN exim-4.10/src/globals.h exim-new/src/globals.h
--- exim-4.10/src/globals.h    2002-07-22 01:59:49.000000000 -0700
+++ exim-new/src/globals.h    2002-09-19 19:47:40.000000000 -0700
@@ -134,6 +134,8 @@
  extern uschar *bounce_sender_authentication; /* AUTH address for
bounces */
  extern int     bsmtp_transaction_linecount; /* Start of last
transaction */


+extern int     callout_cache_positive_expire; /* Time for positive
callout cache records to expire */
+extern int     callout_cache_negative_expire; /* Time for negative
callout cache records to expire */
  extern uschar *check_dns_names_pattern;/* Regex for syntax check */
  extern int     check_log_inodes;       /* Minimum for message
acceptance */
  extern int     check_log_space;        /* Minimum for message
acceptance */
diff -X /lfs/dontdiff -urN exim-4.10/src/readconf.c exim-new/src/readconf.c
--- exim-4.10/src/readconf.c    2002-07-22 01:59:50.000000000 -0700
+++ exim-new/src/readconf.c    2002-09-19 19:47:40.000000000 -0700
@@ -55,6 +55,8 @@
    { "bounce_message_text",      opt_stringptr,   &bounce_message_text },
    { "bounce_return_message",    opt_bool,        &bounce_return_message },
    {
"bounce_sender_authentication",opt_stringptr,&bounce_sender_authentication
},
+  { "callout_positive_expire",  opt_time,
&callout_cache_positive_expire },
+  { "callout_negative_expire",  opt_time,
&callout_cache_negative_expire },
    { "check_log_inodes",         opt_int,         &check_log_inodes },
    { "check_log_space",          opt_int,         &check_log_space },
    { "check_spool_inodes",       opt_int,         &check_spool_inodes },
diff -X /lfs/dontdiff -urN exim-4.10/src/verify.c exim-new/src/verify.c
--- exim-4.10/src/verify.c    2002-07-22 01:59:52.000000000 -0700
+++ exim-new/src/verify.c    2002-09-19 21:13:05.000000000 -0700
@@ -78,6 +78,10 @@
  address_item *addr_succeed = NULL;
  uschar *ko_prefix, *cr;
  uschar *address = addr->orig;
+open_db dbblock;
+open_db *dbm_file;
+dbdata_callout_cache *cache_record;
+dbdata_callout_cache new_record;


  /* Set up a prefix and suffix for error message which allow us to use
the same
  output statements both in EXPN mode (where an SMTP response is needed)
and when
@@ -210,12 +214,54 @@
      if (host_checking)
        {
        HDEBUG(D_verify) debug_printf("... callout omitted when host
testing\n");
-      done = TRUE;
+      /* done = TRUE; */
        }


-    else for (host = addr->host_list;
-              host != NULL && ! done;
-              host = host->next)
+    /* Open the callout cache database, creating it if there isn't one.
Get any
+    cache record for this address, and if it has expired, remove it
from the
+    cache. */
+
+    cache_record = NULL;
+    dbm_file = NULL;
+    if ((dbm_file = dbfn_open(US"callout", O_RDONLY, &dbblock, FALSE))
== NULL)
+      {
+    DEBUG(D_verify)
+      debug_printf("no callout cache available\n");
+      }
+    if (dbm_file != NULL)
+      {
+    DEBUG(D_verify)
+      debug_printf("callout cache opened\n");
+    cache_record = dbfn_read(dbm_file, addr->rcpt);
+    if (cache_record != NULL)
+      {
+        DEBUG(D_verify)
+          debug_printf("got callout cache record\n");
+           if ((time(NULL) - cache_record->time_stamp) >
+           (cache_record->positive ? callout_cache_positive_expire :
callout_cache_negative_expire))
+         {
+           if (dbfn_delete(dbm_file, addr->rcpt) != 0)
+         {
+           DEBUG(D_verify)
+             debug_printf("could not remove callout cache record\n");
+         }
+           cache_record = NULL;
+         }
+      }
+      }
+
+    if (cache_record != NULL)
+      {
+    if (!cache_record->positive)
+      rc = FAIL;
+    done = TRUE;
+    DEBUG(D_verify)
+          debug_printf("used callout cache %s record\n",
cache_record->positive ? "positive" : "negative");
+      }
+
+    for (host = addr->host_list;
+     host != NULL && ! done;
+     host = host->next)
        {
        smtp_inblock inblock;
        smtp_outblock outblock;
@@ -345,6 +391,20 @@
        close(inblock.sock);
        }    /* Loop through all hosts, while !done */


+    /* Cache the callout result, if it was not a deferral, and if there was
+    no previous cache entry for this address. */
+    if (done && (cache_record == NULL) && (dbm_file != NULL))
+      {
+    int result;
+
+    new_record.positive = (rc != FAIL);
+    result = dbfn_write(dbm_file, addr->rcpt, &new_record,
(int)sizeof(dbdata_callout_cache));
+    DEBUG(D_verify)
+      debug_printf("dbfn_write result: %d\n", result);
+    DEBUG(D_verify)
+      debug_printf("wrote %s callout cache record\n", new_record.positive
? "positive" : "negative");
+      }
+
      /* Failure to connect to any host, or any response other than 2xx
or 5xx
      is a temporary error. If there was only one host, and a response was
      received, leave it alone. Otherwise, give a generic response. */
@@ -358,6 +418,9 @@
              is_recipient? "recipient" : "sender");
        addr->basic_errno = ERRNO_CALLOUTDEFER;
        }
+    if (dbm_file != NULL)
+      dbfn_close(dbm_file);
+
      }   /* Callout */


    /* A router may return REROUTE if it has changed the value of
addr->domain