Gitweb:
https://git.exim.org/exim.git/commitdiff/5914065c4651621ed7a5b832094ed9fe83bb2ca4
Commit: 5914065c4651621ed7a5b832094ed9fe83bb2ca4
Parent: 0e4b2a6e28fec7514397d4341c6c4e4190523e2c
Author: Jeremy Harris <jgh146exb@???>
AuthorDate: Sat Jul 27 14:17:12 2024 +0100
Committer: Jeremy Harris <jgh146exb@???>
CommitDate: Sat Jul 27 14:17:12 2024 +0100
Fix dmbjz on sqlite
---
doc/doc-txt/ChangeLog | 5 ++
src/OS/Makefile-Base | 21 +++++--
src/src/dbfn.c | 2 +-
src/src/hintsdb/hints_sqlite.h | 129 ++++++++++++++++++++++++++++-------------
src/src/string.c | 4 +-
src/src/xclient.c | 4 +-
src/src/xtextencode.c | 43 ++++++++++++--
test/confs/0195 | 2 +-
test/scripts/0000-Basic/0195 | 9 +++
9 files changed, 161 insertions(+), 58 deletions(-)
diff --git a/doc/doc-txt/ChangeLog b/doc/doc-txt/ChangeLog
index 2a7a70c47..7a4f1757e 100644
--- a/doc/doc-txt/ChangeLog
+++ b/doc/doc-txt/ChangeLog
@@ -22,6 +22,11 @@ JH/03 With dkim_verify_minimal, avoid calling the DKIM ACL after the first
JH/04 Remove the docs and support scripts dealing with conversion of Exim
version 3 installations.
+JH/05 Fix hintsdb support for dbmjz when compiled using sqlite3. Previously
+ the backend support assumed keys would be simple C strings, but dbmjz
+ uses keys with embedded NUL bytes. The builtin hintsdb use is unaffected,
+ but installations using dbmjz will need to rebuild those DBs.
+
Exim version 4.98
-----------------
diff --git a/src/OS/Makefile-Base b/src/OS/Makefile-Base
index 4df9451f8..f494249bf 100644
--- a/src/OS/Makefile-Base
+++ b/src/OS/Makefile-Base
@@ -544,7 +544,7 @@ exim: buildlookups buildauths pdkim/pdkim.a \
# The utility for dumping the contents of an exim database
-OBJ_DUMPDB = exim_dumpdb.o util-os.o util-store.o
+OBJ_DUMPDB = exim_dumpdb.o util-os.o util-store.o util-xtextencode.o
exim_dumpdb: $(OBJ_DUMPDB)
@echo "$(LNCC) -o exim_dumpdb"
@@ -559,7 +559,7 @@ exim_dumpdb: $(OBJ_DUMPDB)
# The utility for interrogating/fixing the contents of an exim database
-OBJ_FIXDB = exim_fixdb.o util-os.o util-store.o util-md5.o
+OBJ_FIXDB = exim_fixdb.o util-os.o util-store.o util-md5.o util-xtextencode.o
exim_fixdb: $(OBJ_FIXDB)
@echo "$(LNCC) -o exim_fixdb"
@@ -574,11 +574,12 @@ exim_fixdb: $(OBJ_FIXDB)
# The utility for tidying the contents of an exim database
-OBJ_TIDYDB = exim_tidydb.o util-os.o util-store.o
+OBJ_TIDYDB = exim_tidydb.o util-os.o util-store.o util-xtextencode.o
exim_tidydb: $(OBJ_TIDYDB)
@echo "$(LNCC) -o exim_tidydb"
- $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_tidydb $(LFLAGS) $(OBJ_TIDYDB) \
+ $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_tidydb $(LFLAGS) \
+ $(OBJ_TIDYDB) \
$(LIBS) $(EXTRALIBS) $(DBMLIB)
@if [ x"$(STRIP_COMMAND)" != x"" ]; then \
echo $(STRIP_COMMAND) exim_tidydb; \
@@ -589,9 +590,12 @@ exim_tidydb: $(OBJ_TIDYDB)
# The utility for building dbm files
-exim_dbmbuild: exim_dbmbuild.o
+OBJ_DBMBUILD = exim_dbmbuild.o util-xtextencode.o
+
+exim_dbmbuild: $(OBJ_DBMBUILD)
@echo "$(LNCC) -o exim_dbmbuild"
- $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_dbmbuild $(LFLAGS) exim_dbmbuild.o \
+ $(FE)$(LNCC) $(CFLAGS) $(INCLUDE) -o exim_dbmbuild $(LFLAGS) \
+ $(OBJ_DBMBUILD) \
$(LIBS) $(EXTRALIBS) $(DBMLIB)
@if [ x"$(STRIP_COMMAND)" != x"" ]; then \
echo $(STRIP_COMMAND) exim_dbmbuild; \
@@ -636,6 +640,7 @@ OBJ_MONBIN = util-host_address.o \
util-string.o \
util-tod.o \
util-tree.o \
+ util-xtextencode.o \
$(MONBIN)
eximon.bin: $(EXIMON_EDITME) eximon $(OBJ_MONBIN) ../exim_monitor/em_version.c \
@@ -794,6 +799,10 @@ util-tree.o: $(HDRS) tree.c
@echo "$(CC) -DCOMPILE_UTILITY tree.c"
$(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-tree.o tree.c
+util-xtextencode.o: $(HDRS) xtextencode.c
+ @echo "$(CC) -DCOMPILE_UTILITY xtextencode.c"
+ $(FE)$(CC) -c $(CFLAGS) $(INCLUDE) -DCOMPILE_UTILITY -o util-xtextencode.o xtextencode.c
+
util-os.o: $(HDRS) os.c
@echo "$(CC) -DCOMPILE_UTILITY os.c"
$(FE)$(CC) -c $(CFLAGS) $(INCLUDE) \
diff --git a/src/src/dbfn.c b/src/src/dbfn.c
index b9eef503f..1d24769db 100644
--- a/src/src/dbfn.c
+++ b/src/src/dbfn.c
@@ -600,7 +600,7 @@ yield = exim_dbscan(dbblock->dbptr, &key_datum, &value_datum, start, *cursor)
if (!yield) exim_dbdelete_cursor(*cursor);
return yield;
}
-#endif
+#endif /*notdef*/
diff --git a/src/src/hintsdb/hints_sqlite.h b/src/src/hintsdb/hints_sqlite.h
index d90c2b878..6ef214521 100644
--- a/src/src/hintsdb/hints_sqlite.h
+++ b/src/src/hintsdb/hints_sqlite.h
@@ -25,7 +25,13 @@ backend provider. */
/* Some text for messages */
# define EXIM_DBTYPE "sqlite3"
-# /* Access functions */
+/* Utility functionss */
+
+extern uschar *xtextencode(uschar *, int);
+extern int xtextdecode(uschar *, uschar**);
+
+
+/* Access functions */
static inline BOOL
exim_lockfile_needed(void)
@@ -86,15 +92,17 @@ sqlite3_stmt * statement;
int ret;
res->len = (size_t) -1;
-/* fprintf(stderr, "exim_dbget__(%s)\n", s); */
+/* DEBUG(D_hints_lookup) debug_printf_indent("exim_dbget__(%s)\n", s); */
if ((ret = sqlite3_prepare_v2(dbp, CCS s, -1, &statement, NULL)) != SQLITE_OK)
{
-/* fprintf(stderr, "prepare fail: %s\n", sqlite3_errmsg(dbp)); */
+ DEBUG(D_hints_lookup)
+ debug_printf_indent("prepare fail: %s\n", sqlite3_errmsg(dbp));
return FALSE;
}
if (sqlite3_step(statement) != SQLITE_ROW)
{
-/* fprintf(stderr, "step fail: %s\n", sqlite3_errmsg(dbp)); */
+ /* DEBUG(D_hints_lookup)
+ debug_printf_indent("step fail: %s\n", sqlite3_errmsg(dbp)); */
sqlite3_finalize(statement);
return FALSE;
}
@@ -108,7 +116,8 @@ res->data = store_get(res->len +1, GET_TAINTED);
# endif
memcpy(res->data, sqlite3_column_blob(statement, 0), res->len);
res->data[res->len] = '\0';
-/* fprintf(stderr, "res %d bytes: '%.*s'\n", (int)res->len, (int)res->len, res->data); */
+/* DEBUG(D_hints_lookup) debug_printf_indent("res %d bytes: '%.*s'\n",
+ (int)res->len, (int)res->len, res->data); */
sqlite3_finalize(statement);
return TRUE;
}
@@ -116,22 +125,30 @@ return TRUE;
static inline BOOL
exim_dbget(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res)
{
-# define FMT "SELECT dat FROM tbl WHERE ky = '%.*s';"
-uschar * qry;
+# define FMT "SELECT dat FROM tbl WHERE ky = '%s';"
+uschar * encoded_key, * qry;
int i;
BOOL ret;
# ifdef COMPILE_UTILITY
-/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */
-i = snprintf(NULL, 0, FMT, (int) key->len, key->data)+1;
+if (!(encoded_key = xtextencode(key->data, key->len)))
+ return FALSE;
+# else
+encoded_key = xtextencode(key->data, key->len);
+# endif
+/* DEBUG(D_hints_lookup) debug_printf_indent("exim_dbget(k len %d '%s')\n",
+ (int)key->len, encoded_key); */
+
+# ifdef COMPILE_UTILITY
+i = snprintf(NULL, 0, FMT, encoded_key) + 1;
if (!(qry = malloc(i)))
return FALSE;
-snprintf(CS qry, i, FMT, (int) key->len, key->data);
+snprintf(CS qry, i, FMT, encoded_key);
ret = exim_dbget__(dbp, qry, res);
free(qry);
+free(encoded_key);
# else
-/* fprintf(stderr, "exim_dbget(k len %d '%.*s')\n", (int)key->len, (int)key->len, key->data); */
-qry = string_sprintf(FMT, (int) key->len, key->data);
+qry = string_sprintf(FMT, encoded_key);
ret = exim_dbget__(dbp, qry, res);
# endif
@@ -140,7 +157,11 @@ return ret;
}
/* Note that we return claiming a duplicate record for any error.
-It seem not uncommon to get a "database is locked" error. */
+It seem not uncommon to get a "database is locked" error.
+
+Keys are stored xtext-encoded (which is mostly readable, for plaintext).
+Values are stored in a BLOB type in the DB, for which the SQL interface
+is hex-encoded. */
# define EXIM_DBPUTB_OK 0
# define EXIM_DBPUTB_DUP (-1)
@@ -148,8 +169,8 @@ static inline int
exim_s_dbp(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data, const uschar * alt)
{
int hlen = data->len * 2, off = 0, res;
-# define FMT "INSERT OR %s INTO tbl (ky,dat) VALUES ('%.*s', X'%.*s');"
-uschar * qry;
+# define FMT "INSERT OR %s INTO tbl (ky,dat) VALUES ('%s', X'%.*s');"
+uschar * encoded_key, * qry;
# ifdef COMPILE_UTILITY
uschar * hex = malloc(hlen+1);
if (!hex) return EXIM_DBPUTB_DUP; /* best we can do */
@@ -157,27 +178,37 @@ if (!hex) return EXIM_DBPUTB_DUP; /* best we can do */
uschar * hex = store_get(hlen+1, data->data);
# endif
+/* Encode the value for the SQL API */
+
for (const uschar * s = data->data, * t = s + data->len; s < t; s++, off += 2)
sprintf(CS hex + off, "%02X", *s);
# ifdef COMPILE_UTILITY
-res = snprintf(CS hex, 0, FMT, alt, (int) key->len, key->data, hlen, hex) +1;
+if (!(encoded_key = xtextencode(key->data, key->len)))
+ return EXIM_DBPUTB_DUP;
+res = snprintf(CS hex, 0, FMT, alt, encoded_key, hlen, hex) +1;
if (!(qry = malloc(res))) return EXIM_DBPUTB_DUP;
-snprintf(CS qry, res, FMT, alt, (int) key->len, key->data, hlen, hex);
-/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */
+snprintf(CS qry, res, FMT, alt, encoded_key, hlen, hex);
+DEBUG(D_hints_lookup) debug_printf_indent("exim_s_dbp(%s)\n", qry);
+
res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL);
free(qry);
+free(encoded_key);
free(hex);
+
# else
-qry = string_sprintf(FMT, alt, (int) key->len, key->data, hlen, hex);
-/* fprintf(stderr, "exim_s_dbp(%s)\n", qry); */
+encoded_key = xtextencode(key->data, key->len);
+qry = string_sprintf(FMT, alt, encoded_key, hlen, hex);
+/* DEBUG(D_hints_lookup) debug_printf_indent("exim_s_dbp(%s)\n", qry); */
+
res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL);
-/* fprintf(stderr, "exim_s_dbp res %d\n", res); */
+/* DEBUG(D_hints_lookup) debug_printf_indent("exim_s_dbp res %d\n", res); */
# endif
# ifdef COMPILE_UTILITY
if (res != SQLITE_OK)
- fprintf(stderr, "sqlite3_exec: %s\n", sqlite3_errmsg(dbp));
+ DEBUG(D_hints_lookup)
+ debug_printf_indent("sqlite3_exec: %s\n", sqlite3_errmsg(dbp));
# endif
return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP;
@@ -189,7 +220,7 @@ return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP;
static inline int
exim_dbput(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * data)
{
-/* fprintf(stderr, "exim_dbput()\n"); */
+/* DEBUG(D_hints_lookup) debug_printf_indent("exim_dbput()\n"); */
(void) exim_s_dbp(dbp, key, data, US"REPLACE");
return 0;
}
@@ -208,22 +239,27 @@ return exim_s_dbp(dbp, key, data, US"ABORT");
static inline int
exim_dbdel(EXIM_DB * dbp, EXIM_DATUM * key)
{
-# define FMT "DELETE FROM tbl WHERE ky = '%.*s';"
-uschar * qry;
+# define FMT "DELETE FROM tbl WHERE ky = '%s';"
+uschar * encoded_key, * qry;
int res;
# ifdef COMPILE_UTILITY
-res = snprintf(NULL, 0, FMT, (int) key->len, key->data) +1; /* res includes nul */
+if (!(encoded_key = xtextencode(key->data, key->len)))
+ return EXIM_DBPUTB_DUP;
+res = snprintf(NULL, 0, FMT, encoded_key) +1; /* res includes nul */
if (!(qry = malloc(res))) return SQLITE_NOMEM;
-snprintf(CS qry, res, FMT, (int) key->len, key->data);
+snprintf(CS qry, res, FMT, encoded_key);
res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL);
free(qry);
+
# else
-qry = string_sprintf(FMT, (int) key->len, key->data);
+encoded_key = xtextencode(key->data, key->len);
+qry = string_sprintf(FMT, encoded_key);
res = sqlite3_exec(dbp, CS qry, NULL, NULL, NULL);
+
# endif
-return res;
+return res == SQLITE_OK ? EXIM_DBPUTB_OK : EXIM_DBPUTB_DUP;
# undef FMT
}
@@ -245,31 +281,42 @@ return c;
}
/* EXIM_DBSCAN */
-/* Note that we return the (next) key, not the record value */
+/* Note that we return the (next) key, not the record value.
+We allocate memory for the return. */
+
static inline BOOL
exim_dbscan(EXIM_DB * dbp, EXIM_DATUM * key, EXIM_DATUM * res, BOOL first,
EXIM_CURSOR * cursor)
{
# define FMT "SELECT ky FROM tbl ORDER BY ky LIMIT 1 OFFSET %d;"
uschar * qry;
-int i;
+EXIM_DATUM encoded_key;
BOOL ret;
# ifdef COMPILE_UTILITY
-i = snprintf(NULL, 0, FMT, *cursor)+1;
+int i = snprintf(NULL, 0, FMT, *cursor)+1;
+
if (!(qry = malloc(i))) return FALSE;
snprintf(CS qry, i, FMT, *cursor);
-/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */
-ret = exim_dbget__(dbp, qry, key);
+DEBUG(D_hints_lookup) debug_printf_indent("exim_dbscan(%s)\n", qry);
+ret = exim_dbget__(dbp, qry, &encoded_key);
free(qry);
-/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */
-# else
+
+# else /*!COMPILE_UTILITY*/
qry = string_sprintf(FMT, *cursor);
-/* fprintf(stderr, "exim_dbscan(%s)\n", qry); */
-ret = exim_dbget__(dbp, qry, key);
-/* fprintf(stderr, "exim_dbscan ret %c\n", ret ? 'T':'F'); */
-# endif
-if (ret) *cursor = *cursor + 1;
+DEBUG(D_hints_lookup) debug_printf_indent("exim_dbscan(%s)\n", qry);
+ret = exim_dbget__(dbp, qry, &encoded_key);
+
+# endif /*COMPILE_UTILITY*/
+
+DEBUG(D_hints_lookup)
+ debug_printf_indent("exim_dbscan ret %c\n", ret ? 'T':'F');
+
+if (ret)
+ {
+ key->len = xtextdecode(encoded_key.data, &key->data);
+ *cursor = *cursor + 1;
+ }
return ret;
# undef FMT
}
diff --git a/src/src/string.c b/src/src/string.c
index aa768d03c..5583d93de 100644
--- a/src/src/string.c
+++ b/src/src/string.c
@@ -1626,7 +1626,7 @@ while (*fp)
case 'W': /* Maybe mark up ctrls, spaces & newlines */
s = va_arg(ap, char *);
- if (Ustrpbrk(s, " \n") && !IS_DEBUG(D_noutf8))
+ if (s && !IS_DEBUG(D_noutf8))
{
gstring * zg = NULL;
int p = precision;
@@ -1660,7 +1660,7 @@ while (*fp)
}
}
if (zg) { s = CS zg->s; slen = gstring_length(zg); }
- else { s = null; slen = Ustrlen(s); }
+ else { s = ""; slen = 0; }
}
else
{
diff --git a/src/src/xclient.c b/src/src/xclient.c
index af2f287b9..af7a94b2d 100644
--- a/src/src/xclient.c
+++ b/src/src/xclient.c
@@ -10,7 +10,9 @@
#ifdef EXPERIMENTAL_XCLIENT
-/* From https://www.postfix.org/XCLIENT_README.html I infer two generations of
+/* This is a proxy protocol.
+
+From https://www.postfix.org/XCLIENT_README.html I infer two generations of
protocol. The more recent one obviates the utility of the HELO attribute, since
it mandates the proxy always sending a HELO/EHLO smtp command following (a
successful) XCLIENT command, and that will carry a NELO name (which we assume,
diff --git a/src/src/xtextencode.c b/src/src/xtextencode.c
index fa2b26bed..71489bd4f 100644
--- a/src/src/xtextencode.c
+++ b/src/src/xtextencode.c
@@ -26,8 +26,9 @@ Returns: a pointer to the zero-terminated xtext string, which
is in working store
*/
+#ifndef COMPILE_UTILITY
uschar *
-xtextencode(uschar *clear, int len)
+xtextencode(uschar * clear, int len)
{
gstring * g = NULL;
for(uschar ch; len > 0; len--, clear++)
@@ -38,6 +39,31 @@ gstring_release_unused(g);
return string_from_gstring(g);
}
+#else /*COMPILE_UTILITY*/
+uschar *
+xtextencode(uschar * clear, int len)
+{
+int enc_len = 1, i = len; /* enc_len includes space for terminating NUL */
+uschar * yield, * s;
+
+for (s = clear; i; i--, s++)
+ {
+ uschar ch = *s;
+ enc_len += ch < 33 || ch > 126 || ch == '+' || ch == '='
+ ? 3 : 1;
+ }
+if (!(yield = s = malloc(enc_len)))
+ return NULL;
+for(uschar ch; len > 0; len--, clear++)
+ if ((ch = *clear) < 33 || ch > 126 || ch == '+' || ch == '=')
+ s += sprintf(CS s, "+%.02X", ch);
+ else
+ *s++ = ch;
+*s = '\0';
+return yield;
+}
+
+#endif /*COMPILE_UTILITY*/
/*************************************************
* Decode byte-string in xtext *
@@ -64,24 +90,29 @@ int
xtextdecode(uschar * code, uschar ** ptr)
{
int x;
+#ifdef COMPILE_UTILITY
+uschar * result = malloc(Ustrlen(code) + 1);
+#else
uschar * result = store_get(Ustrlen(code) + 1, code);
-*ptr = result;
+#endif
-while ((x = (*code++)) != 0)
+*ptr = result;
+while ((x = (*code++)))
{
if (x < 33 || x > 127 || x == '=') return -1;
if (x == '+')
{
- register int y;
+ int y;
if (!isxdigit((x = (*code++)))) return -1;
y = ((isdigit(x))? x - '0' : (tolower(x) - 'a' + 10)) << 4;
if (!isxdigit((x = (*code++)))) return -1;
*result++ = y | ((isdigit(x))? x - '0' : (tolower(x) - 'a' + 10));
}
- else *result++ = x;
+ else
+ *result++ = x;
}
-*result = 0;
+*result = '\0';
return result - *ptr;
}
diff --git a/test/confs/0195 b/test/confs/0195
index 46f1029ef..fb69964d1 100644
--- a/test/confs/0195
+++ b/test/confs/0195
@@ -7,7 +7,7 @@ primary_hostname = myhost.test.ex
# ----- Main settings -----
domainlist local_domains = test.ex : *.test.ex
-
+queue_run_in_order
# ----- Routers -----
diff --git a/test/scripts/0000-Basic/0195 b/test/scripts/0000-Basic/0195
index 3c487ded1..5ce0b3e94 100644
--- a/test/scripts/0000-Basic/0195
+++ b/test/scripts/0000-Basic/0195
@@ -1,21 +1,30 @@
# retry times on local addresses
+#
+# All three should be deferred
exim -odi userx usery userz
This is a test message.
****
+# retry records should have been created
dump retry
sleep 1
+# retry record should mean no delivery tried
exim -R usery
****
+# should still be all three records
dump retry
sleep 1
+# insert another message; should be deferred only on first delivery try
exim -odi usery
one-defer: set
second message
****
+# all three records should still be there
dump retry
sleep 1
+# msg2 has had it's single-defer so should get delivered
exim -Mc $msg2
****
+# the record for that localpart should have been removed as a result
dump retry
sleep 1
exim -q
--
## subscription configuration (requires account):
##
https://lists.exim.org/mailman3/postorius/lists/exim-cvs.lists.exim.org/
## unsubscribe (doesn't require an account):
## exim-cvs-unsubscribe@???
## Exim details at
http://www.exim.org/
## Please use the Wiki with this list -
http://wiki.exim.org/
