Gitweb:
https://git.exim.org/exim.git/commitdiff/b2a6e91b334223c00d07dd3a7ca6d71626d21bf3
Commit: b2a6e91b334223c00d07dd3a7ca6d71626d21bf3
Parent: ac8aeb5485a80a06ac8a52b43b84210564cd7e09
Author: Jeremy Harris <jgh146exb@???>
AuthorDate: Sat Aug 24 22:57:07 2024 +0100
Committer: Jeremy Harris <jgh146exb@???>
CommitDate: Sat Aug 24 22:57:07 2024 +0100
Lookups: lazy-load modules
---
doc/doc-docbook/spec.xfpt | 5 +-
doc/doc-txt/ChangeLog | 5 ++
src/src/drtables.c | 151 +++++++++++++++++++++++++++++-----------------
src/src/exim.c | 4 ++
src/src/functions.h | 6 ++
src/src/match.c | 3 +-
src/src/readconf.c | 6 ++
src/src/search.c | 42 ++++++++++---
src/src/store.c | 2 +-
src/src/structs.h | 2 +-
src/src/verify.c | 12 ++--
test/runtest | 11 ++--
12 files changed, 166 insertions(+), 83 deletions(-)
diff --git a/doc/doc-docbook/spec.xfpt b/doc/doc-docbook/spec.xfpt
index 4728d9e78..600eee4af 100644
--- a/doc/doc-docbook/spec.xfpt
+++ b/doc/doc-docbook/spec.xfpt
@@ -2084,7 +2084,10 @@ library dependencies without requiring all systems to install all of those
dependencies.
.new
Any combination of lookup types can be built this way.
-All of the lookup modules found as an Exim process starts will be loaded.
+Lookup types that provide several variants will be loaded as
+Exim starts.
+Types that provide only one method are not loaded until used by
+the runtime configuration.
.wen
For building
diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 0cefe5b7c..22c934b82 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -51,6 +51,11 @@ JH/10 Bug 3108: On platforms not providing strchrnul() [OpenBSD] supply a proper
different size. This resulted in crashes while processing DKIM signatures
of received messages. Identification and fix from Qualys Security.
+JH/11 Lookups built as dynamic-load modules which support a single lookup
+ type are now only loaded if required by the config. Previously all lookup
+ modules present in the modules directory were loaded; this now applies
+ only to those supporting multiple types.
+
Exim version 4.98
-----------------
diff --git a/src/src/drtables.c b/src/src/drtables.c
index 0c3dd7049..a144085c5 100644
--- a/src/src/drtables.c
+++ b/src/src/drtables.c
@@ -340,14 +340,90 @@ extern lookup_module_info whoson_lookup_module_info;
extern lookup_module_info readsock_lookup_module_info;
+#ifdef LOOKUP_MODULE_DIR
+/* Try to load a lookup module with the given name.
+
+Arguments:
+ name name of the lookup
+ errstr if not NULL, place "open fail" error message here
+
+Return: boolean success
+*/
+
+static BOOL
+lookup_mod_load(const uschar * name, uschar ** errstr)
+{
+const uschar * path = string_sprintf(
+ LOOKUP_MODULE_DIR "/%s_lookup." DYNLIB_FN_EXT, name);
+void * dl;
+struct lookup_module_info * info;
+const char * errormsg;
+
+if (!(dl = dlopen(CS path, RTLD_NOW)))
+ {
+ if (errstr)
+ *errstr = string_sprintf("Error loading %s: %s", name, dlerror());
+ else
+ (void) dlerror(); /* clear out error state */
+ return FALSE;
+ }
+
+/* FreeBSD nsdispatch() can trigger dlerror() errors about
+_nss_cache_cycle_prevention_function; we need to clear the dlerror()
+state before calling dlsym(), so that any error afterwards only comes
+from dlsym(). */
+
+errormsg = dlerror();
+
+info = (struct lookup_module_info *) dlsym(dl, "_lookup_module_info");
+if ((errormsg = dlerror()))
+ {
+ fprintf(stderr, "%s does not appear to be a lookup module (%s)\n", name, errormsg);
+ log_write(0, LOG_MAIN|LOG_PANIC, "%s does not appear to be a lookup module (%s)", name, errormsg);
+ dlclose(dl);
+ return FALSE;
+ }
+if (info->magic != LOOKUP_MODULE_INFO_MAGIC)
+ {
+ fprintf(stderr, "Lookup module %s is not compatible with this version of Exim\n", name);
+ log_write(0, LOG_MAIN|LOG_PANIC, "Lookup module %s is not compatible with this version of Exim", name);
+ dlclose(dl);
+ return FALSE;
+ }
+
+addlookupmodule(info);
+DEBUG(D_lookup) debug_printf_indent("Loaded \"%s\" (%d lookup type%s)\n",
+ name, info->lookupcount,
+ info->lookupcount > 1 ? "s" : "");
+return TRUE;
+}
+
+
+/* Try to load a lookup module, assuming the module name is the same
+as the lookup type name. This will only work for single-method modules.
+Other have to be always-load (see the RE in init_lookup_list() below).
+*/
+
+BOOL
+lookup_one_mod_load(const uschar * name, uschar ** errstr)
+{
+if (!lookup_mod_load(name, errstr)) return FALSE;
+/*XXX notify daemon? */
+return TRUE;
+}
+
+#endif /*LOOKUP_MODULE_DIR*/
+
+
+
+
+
void
init_lookup_list(void)
{
#ifdef LOOKUP_MODULE_DIR
DIR * dd;
-struct dirent * ent;
int countmodules = 0;
-int moduleerrors = 0;
#endif
static BOOL lookup_list_init_done = FALSE;
@@ -441,6 +517,9 @@ implemented by a lookup module. */
addlookupmodule(&readsock_lookup_module_info);
+DEBUG(D_lookup) debug_printf("Total %d built-in lookups\n", lookup_list_count);
+
+
#ifdef LOOKUP_MODULE_DIR
if (!(dd = exim_opendir(CUS LOOKUP_MODULE_DIR)))
{
@@ -450,81 +529,39 @@ if (!(dd = exim_opendir(CUS LOOKUP_MODULE_DIR)))
}
else
{
+ /* Look specifically for modules we know offer several lookup types and
+ load them now, since we cannot load-on-first-use. */
+
+ struct dirent * ent;
const pcre2_code * regex_islookupmod = regex_must_compile(
- US"_lookup\\." DYNLIB_FN_EXT "$", MCS_NOFLAGS, TRUE);
+ US"(lsearch|ldap|nis)_lookup\\." DYNLIB_FN_EXT "$", MCS_NOFLAGS, TRUE);
DEBUG(D_lookup) debug_printf("Loading lookup modules from %s\n", LOOKUP_MODULE_DIR);
while ((ent = readdir(dd)))
{
char * name = ent->d_name;
int len = (int)strlen(name);
- if (regex_match(regex_islookupmod, US name, len, NULL))
+ if (regex_match_and_setup(regex_islookupmod, US name, 0, 0))
{
- int pathnamelen = len + (int)strlen(LOOKUP_MODULE_DIR) + 2;
- void *dl;
- struct lookup_module_info *info;
- const char *errormsg;
-
- /* SRH: am I being paranoid here or what? */
- if (pathnamelen > big_buffer_size)
- {
- fprintf(stderr, "Loading lookup modules: %s/%s: name too long\n", LOOKUP_MODULE_DIR, name);
- log_write(0, LOG_MAIN|LOG_PANIC, "%s/%s: name too long\n", LOOKUP_MODULE_DIR, name);
- continue;
- }
-
- /* SRH: snprintf here? */
- sprintf(CS big_buffer, "%s/%s", LOOKUP_MODULE_DIR, name);
-
- if (!(dl = dlopen(CS big_buffer, RTLD_NOW)))
+ uschar * errstr;
+ if (lookup_mod_load(expand_nstring[1], &errstr))
+ countmodules++;
+ else
{
- errormsg = dlerror();
- fprintf(stderr, "Error loading %s: %s\n", name, errormsg);
- log_write(0, LOG_MAIN|LOG_PANIC, "Error loading lookup module %s: %s\n", name, errormsg);
- moduleerrors++;
- continue;
+ fprintf(stderr, "%s\n", errstr);
+ log_write(0, LOG_MAIN|LOG_PANIC, "%s", errstr);
}
-
- /* FreeBSD nsdispatch() can trigger dlerror() errors about
- _nss_cache_cycle_prevention_function; we need to clear the dlerror()
- state before calling dlsym(), so that any error afterwards only comes
- from dlsym(). */
-
- errormsg = dlerror();
-
- info = (struct lookup_module_info*) dlsym(dl, "_lookup_module_info");
- if ((errormsg = dlerror()))
- {
- fprintf(stderr, "%s does not appear to be a lookup module (%s)\n", name, errormsg);
- log_write(0, LOG_MAIN|LOG_PANIC, "%s does not appear to be a lookup module (%s)\n", name, errormsg);
- dlclose(dl);
- moduleerrors++;
- continue;
- }
- if (info->magic != LOOKUP_MODULE_INFO_MAGIC)
- {
- fprintf(stderr, "Lookup module %s is not compatible with this version of Exim\n", name);
- log_write(0, LOG_MAIN|LOG_PANIC, "Lookup module %s is not compatible with this version of Exim\n", name);
- dlclose(dl);
- moduleerrors++;
- continue;
- }
-
- addlookupmodule(info);
- DEBUG(D_lookup) debug_printf("Loaded \"%s\" (%d lookup types)\n", name, info->lookupcount);
- countmodules++;
}
}
- store_free((void*)regex_islookupmod);
closedir(dd);
}
DEBUG(D_lookup) debug_printf("Loaded %d lookup modules\n", countmodules);
#endif
-DEBUG(D_lookup) debug_printf("Total %d lookups\n", lookup_list_count);
}
+
#endif /*!MACRO_PREDEF*/
/* End of drtables.c */
diff --git a/src/src/exim.c b/src/src/exim.c
index 1a22c9af2..de1f48434 100644
--- a/src/src/exim.c
+++ b/src/src/exim.c
@@ -1806,6 +1806,10 @@ len = Ustrlen(big_buffer);
(void) macros_expand(0, &len, &dummy_macexp);
+#ifdef LOOKUP_MODULE_DIR
+//mod_load_check(big_buffer);
+#endif
+
if (isupper(big_buffer[0]))
{
if (macro_read_assignment(big_buffer))
diff --git a/src/src/functions.h b/src/src/functions.h
index b97895b8a..be60d5fc9 100644
--- a/src/src/functions.h
+++ b/src/src/functions.h
@@ -339,6 +339,9 @@ extern const uschar *local_part_quote(const uschar *);
extern int log_open_as_exim(const uschar * const);
extern void log_close_all(void);
extern const lookup_info * lookup_with_acq_num(unsigned);
+#ifdef LOOKUP_MODULE_DIR
+extern BOOL lookup_one_mod_load(const uschar *, uschar **);
+#endif
extern macro_item * macro_create(const uschar *, const uschar *, BOOL);
@@ -388,6 +391,9 @@ extern void moan_tell_someone(uschar *, address_item *,
extern BOOL moan_to_sender(int, error_block *, header_line *, FILE *, BOOL);
extern void moan_write_from(FILE *);
extern void moan_write_references(FILE *, uschar *);
+#ifdef LOOKUP_MODULE_DIR
+//extern void mod_load_check(const uschar *);
+#endif
extern FILE *modefopen(const uschar *, const char *, mode_t);
extern int open_cutthrough_connection( address_item * addr );
diff --git a/src/src/match.c b/src/src/match.c
index 0b3632f7e..dc88485a4 100644
--- a/src/src/match.c
+++ b/src/src/match.c
@@ -272,7 +272,8 @@ the part of the string preceding the semicolon. */
li = search_findtype_partial(pattern, &partial, &affix, &affixlen,
&starflags, &opts);
*semicolon = ';';
-if (!li) log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message);
+if (!li)
+ log_write(0, LOG_MAIN|LOG_PANIC_DIE, "%s", search_error_message);
/* Partial matching is not appropriate for certain lookups (e.g. when looking
up user@domain for sender rejection). There's a flag to disable it. */
diff --git a/src/src/readconf.c b/src/src/readconf.c
index 532e0774c..a090cfb30 100644
--- a/src/src/readconf.c
+++ b/src/src/readconf.c
@@ -1232,6 +1232,12 @@ if (strncmpic(s, US"begin ", 6) == 0)
return NULL;
}
+#ifdef LOOKUP_MODULE_DIR
+/* Check for any required module load operations */
+
+//mod_load_check(s);
+#endif
+
/* Return the first non-blank character. */
return s;
diff --git a/src/src/search.c b/src/src/search.c
index efa720f25..ad55e8780 100644
--- a/src/src/search.c
+++ b/src/src/search.c
@@ -51,9 +51,17 @@ static rmark search_reset_point = NULL;
* Validate a plain lookup type name *
*************************************************/
+static const lookup_info *
+internal_search_findtype(const uschar * name)
+{
+tree_node * t = tree_search(lookups_tree, name);
+return t ? t->data.ptr : NULL;
+}
+
/* Only those names that are recognized and whose code is included in the
-binary give an OK response. Use a binary chop search now that the list has got
-so long.
+binary give an OK response. Types are held in a binary tree for fast location
+and dynamic insertion. If not initially found, try to load a module if
+any were compiled.
Arguments:
name lookup type name - not necessarily zero terminated (e.g. dbm*)
@@ -66,18 +74,32 @@ Returns: ptr to info struct for the lookup,
const lookup_info *
search_findtype(const uschar * name, int len)
{
-const uschar * s = name[len] ? string_copyn(name, len) : name;
-tree_node * t = tree_search(lookups_tree, s);
+const lookup_info * li;
+
+if (name[len])
+ name = string_copyn(name, len);
+if ((li = internal_search_findtype(name)))
+ return li;
-if (t) return t->data.ptr;
+#ifdef LOOKUP_MODULE_DIR
+ DEBUG(D_lookup)
+ debug_printf_indent("searchtype %s not initially found\n", name);
-search_error_message = string_sprintf("unknown lookup type \"%s\"", s);
+ if (lookup_one_mod_load(name, NULL))
+ if ((li = internal_search_findtype(name)))
+ return li;
+ else
+ { DEBUG(D_lookup) debug_printf_indent("find retry failed\n"); }
+ else DEBUG(D_lookup)
+ debug_printf_indent("scan modules dir for %s failed\n", name);
+#endif
+
+search_error_message = string_sprintf("unknown lookup type \"%s\"", name);
return NULL;
}
-
/*************************************************
* Validate a full lookup type name *
*************************************************/
@@ -207,8 +229,8 @@ Arguments:
Return: keyquery the search-type (for single-key) or query (for query-type)
*/
uschar *
-search_args(const lookup_info * li, uschar * search, uschar * query, uschar ** fnamep,
- const uschar * opts)
+search_args(const lookup_info * li, uschar * search, uschar * query,
+ uschar ** fnamep, const uschar * opts)
{
Uskip_whitespace(&query);
if (mac_islookup(li, lookup_absfilequery))
@@ -430,7 +452,7 @@ if (li->type == lookup_absfile && open_filecount >= lookup_open_max)
((search_cache *)(open_bot->data.ptr))->down = NULL;
else
open_top = NULL;
- ((c->li)->close)(c->handle);
+ (c->li->close)(c->handle);
c->handle = NULL;
open_filecount--;
}
diff --git a/src/src/store.c b/src/src/store.c
index c135b4383..4824b5c54 100644
--- a/src/src/store.c
+++ b/src/src/store.c
@@ -661,7 +661,7 @@ find variants but shared quote functions. */
BOOL
is_quoted_like(const void * p, unsigned quoter)
{
-const uschar * p_name, * q_name;
+const uschar * p_name, * q_name = NULL;
const lookup_info * p_li, * q_li;
void * p_qfn, * q_qfn;
diff --git a/src/src/structs.h b/src/src/structs.h
index 702c7c72c..9492dbac2 100644
--- a/src/src/structs.h
+++ b/src/src/structs.h
@@ -742,7 +742,7 @@ to close. */
typedef struct search_cache {
void *handle; /* lookup handle, or NULL if closed */
- const lookup_info *li; /* info struct for search type */
+ const lookup_info * li; /* info struct for search type */
tree_node *up; /* LRU up pointer */
tree_node *down; /* LRU down pointer */
tree_node *item_cache; /* tree of cached results */
diff --git a/src/src/verify.c b/src/src/verify.c
index f936de50a..05b15858a 100644
--- a/src/src/verify.c
+++ b/src/src/verify.c
@@ -3059,11 +3059,10 @@ else
if (iplookup)
{
- int insize;
const lookup_info * li;
- int incoming[4];
- void *handle;
- uschar *filename, *key, *result;
+ int incoming[4], insize;
+ void * handle;
+ uschar * filename, * key, * result;
uschar buffer[64];
/* Find the search type */
@@ -3094,11 +3093,10 @@ if (iplookup)
}
else /* Single-key style */
{
- int sep = (Ustrcmp(li->name, "iplsearch") == 0)?
- ':' : '.';
+ int sep = Ustrcmp(li->name, "iplsearch") == 0 ? ':' : '.';
insize = host_aton(cb->host_address, incoming);
host_mask(insize, incoming, mlen);
- (void)host_nmtoa(insize, incoming, mlen, buffer, sep);
+ (void) host_nmtoa(insize, incoming, mlen, buffer, sep);
key = buffer;
filename = semicolon + 1;
}
diff --git a/test/runtest b/test/runtest
index e80624484..3e10a5ab7 100755
--- a/test/runtest
+++ b/test/runtest
@@ -1297,11 +1297,12 @@ RESET_AFTER_EXTRA_LINE_READ:
next if /^tls_set_watch\(\) fail on '\/usr\/(?:lib\/ssl|local\/openssl3\/etc\/pki\/tls)\/cert.pem': No such file or directory$/;
# drop lookups
- next if /^$time_pid?(?: Lookups\ \((?:built-in|dynamic)\):
- | Loaded\ "[^.]+\.so"\ \(\d+\ lookup\ types\)
- | Loading\ lookup\ modules\ from
- | Loaded\ \d+\ lookup\ modules
- | Total\ \d+\ lookups)/x;
+ next if /(?: Lookups\ \((?:built-in|dynamic)\):
+ | searchtype\ \w+\ not\ initially\ found
+ | Loaded\ "\w+"\ \(\d+\ lookup\ types?\)
+ | Loading\ lookup\ modules\ from
+ | Loaded\ \d+\ lookup\ modules
+ | Total\ \d+\ built-in\ lookups)/x;
# drop loads of dyn-module drivers
next if /^$time_pid?(?:Loading\ \w+\ (?:router|transport|auth)\ driver\ from
--
## subscription configuration (requires account):
##
https://lists.exim.org/mailman3/postorius/lists/exim-cvs.lists.exim.org/
## unsubscribe (doesn't require an account):
## exim-cvs-unsubscribe@???
## Exim details at
http://www.exim.org/
## Please use the Wiki with this list -
http://wiki.exim.org/
