ph10 2006/02/06 16:07:10 GMT
Added files:
exim-test ABOUT Makefile.in README configure.ac
listtests patchexim runtest
Log:
CVS-ing the new test suite.
Revision Changes Path
1.1 +38 -0 exim/exim-test/ABOUT (new)
1.1 +97 -0 exim/exim-test/Makefile.in (new)
1.1 +1039 -0 exim/exim-test/README (new)
1.1 +54 -0 exim/exim-test/configure.ac (new)
1.1 +67 -0 exim/exim-test/listtests (new)
1.1 +30 -0 exim/exim-test/patchexim (new)
1.1 +3011 -0 exim/exim-test/runtest (new)
Index: ABOUT
====================================================================
$Cambridge: exim/exim-test/ABOUT,v 1.1 2006/02/06 16:07:10 ph10 Exp $
CVS directory exim/exim-test
----------------------------
The files in this directory are those that comprise the Exim test suite. The
README file contains a complete description of the suite and how it works. The
contents of this directory are:
FILES
Makefile.in source of the Makefile for building auxiliary test programs
README description and instructions for running the test suite
configure.ac autoconf configuration file
listtests shell/Perl script for listing the available tests
patchexim Perl script for patching Exim in preparation for testing
runtest Perl script for running the tests
DIRECTORIES
aux-fixed fixed auxiliary data files
aux-var-src source for variable data files
confs Exim configurations for the tests
dnszones-src sources for fake DNS zone files
log saved mainlogs
mail saved mailboxes
msglog saved message logs
paniclog saved panic logs
rejectlog saved reject logs
scripts scripts for the tests
src sources for auxiliary programs
stderr saved stderr outputs
stdout saved stdout output
The scripts directory is subdivided into subdirectories containing tests of
different categories.
End
Index: Makefile.in
====================================================================
# $Cambridge: exim/exim-test/Makefile.in,v 1.1 2006/02/06 16:07:10 ph10 Exp $
# This Makefile builds the support programs for the Exim test suite.
##############################################################################
# These variables are set by the configure script.
CC=@CC@
CFLAGS=@CFLAGS@
LDFLAGS=@LDFLAGS@
CLIENT_SSL=@CLIENT_SSL@
CLIENT_GNUTLS=@CLIENT_GNUTLS@
LOADED=@LOADED@
LOADED_OPT=@LOADED_OPT@
##############################################################################
# List of targets
all: makebin bin/cf bin/client $(CLIENT_SSL) $(CLIENT_GNUTLS) \
bin/checkaccess bin/fakens bin/fd bin/iefbr14 $(LOADED) \
bin/mtpscript bin/server bin/showids
# Ensure the bin directory exists
makebin:; @if [ ! -e bin ] ; then mkdir bin 2>/dev/null; echo ""; fi
# Compile and link the programs:
#
# bin/client is the SMTP script-driven client, without TLS support
# bin/client-ssl is with OpenSSL support
# there isn't yet a version with GnuTLS support
# bin/checkaccess tests whether the exim uid/gid can access the files
# bin/iefbr14 a program that does nothing and returns 0
# bin/loaded is a dynamically loaded test module
# bin/server is the SMTP script-driven server (no TLS support)
bin/cf: src/cf.c
$(CC) $(CFLAGS) $(LDFLAGS) -o bin/cf src/cf.c
@echo ">>> bin/cf command build"
@echo " "
bin/client: src/client.c
$(CC) $(CFLAGS) $(LDFLAGS) -o bin/client src/client.c
@echo ">>> bin/client command built"
@echo " "
bin/client-gnutls: src/client.c
$(CC) $(CFLAGS) -DHAVE_GNUTLS $(LDFLAGS) -lgnutls -lgcrypt -o bin/client-gnutls src/client.c
@echo ">>> bin/client-gnutls command built"
@echo " "
bin/client-ssl: src/client.c
$(CC) $(CFLAGS) -DHAVE_OPENSSL $(LDFLAGS) -lssl -lcrypto -o bin/client-ssl src/client.c
@echo ">>> bin/client-ssl command built"
@echo " "
bin/checkaccess:src/checkaccess.c
$(CC) $(CFLAGS) -DNO_TLS $(LDFLAGS) -o bin/checkaccess src/checkaccess.c
@echo ">>> bin/checkaccess command built"
@echo " "
bin/fakens: src/fakens.c
$(CC) $(CFLAGS) $(LDFLAGS) -o bin/fakens src/fakens.c
@echo ">>> bin/fakens command built"
@echo " "
bin/fd: src/fd.c
$(CC) $(CFLAGS) $(LDFLAGS) -o bin/fd src/fd.c
@echo ">>> bin/fd command built"
@echo " "
bin/iefbr14: src/iefbr14.c
$(CC) $(CFLAGS) $(LDFLAGS) -o bin/iefbr14 src/iefbr14.c
@echo ">>> bin/iefbr14 command built"
@echo " "
bin/loaded: src/loaded.c
$(CC) $(CFLAGS) $(LDFLAGS) $(LOADED_OPT) -o bin/loaded src/loaded.c
@echo ">>> bin/loaded command built"
@echo " "
bin/mtpscript: src/mtpscript.c
$(CC) $(CFLAGS) $(LDFLAGS) $(mtpscript_OPT) -o bin/mtpscript src/mtpscript.c
@echo ">>> bin/mtpscript command built"
@echo " "
bin/server: src/server.c
$(CC) $(CFLAGS) $(LDFLAGS) -o bin/server src/server.c
@echo ">>> bin/server command built"
@echo " "
bin/showids: src/showids.c
$(CC) $(CFLAGS) $(LDFLAGS) -o bin/showids src/showids.c
@echo ">>> bin/showids command built"
@echo " "
# End
Index: README
====================================================================
$Cambridge: exim/exim-test/README,v 1.1 2006/02/06 16:07:10 ph10 Exp $
EXPORTABLE EXIM TEST SUITE
--------------------------
This document last updated for:
Test Suite Version: 4.61
Date: 06 February 2006
BACKGROUND
----------
For a long time, the Exim test suite was confined to Philip Hazel's
workstation, because it relied on that particular environment. The problem is
that an MTA such as Exim interacts a great deal with its environment, so if you
run it somewhere else, the output will be different, which makes automatic
checking difficult. Even in a single environment, things are not all that easy.
For instance, if Exim delivers a message, the log line (which one would want to
compare) contains a timestamp and an Exim message id that will be different
each time. This issue is dealt with by a Perl script that munges the output by
recognizing changing sequences and replacing them with fixed values before
doing a comparison. Another problem with exporting the original test suite is
that it assumes a version of Exim with more or less every optional feature
enabled.
This README describes a new test suite that is intended to be exportable and to
run in a number of different environments. The current status of this project
is "experimental and incomplete". I am releasing it in this state in order to
get feedback on how well it succeeds and of course to iron out any bugs. The
original test suite contains over 600 tests; it will be some time before they
are all re-implemented in the new world.
The tests themselves are in no particular order; they accumulated over the
years as Exim was extended and modified. They vary greatly in size and
complexity. Some were specifically constructed to test new features; others
were made to demonstrate that a bug had been fixed.
A few of the original tests have had to be omitted from this more general
suite because differences in operating system behaviour make it impossible to
generalize them. An example is a test that uses a version of Exim that is
setuid to the Exim user rather than root, with the deliver_drop_privilege
option set. In Linux, such a binary is able to deliver a message as the caller
of Exim, because it can revert to the caller's uid. In FreeBSD this is not the
case.
This is early documentation; it too may be buggy... :-) It is certainly
incomplete, because there are features yet to be added to the test suite.
REQUIREMENTS
------------
In order to run this test suite, the following requirements must be met:
(1) You should run the tests on the latest version of Exim, because the suite
is continuously updated to test the latest features and bug fixes. The
version you test does not, however, have to be installed as the live
version. You can of course run the tests on an older Exim, but some may
fail. In particular, the test suite will fall apart horrible with versions
of Exim prior to 4.54.
(2) You can use any non-root login to run the tests, but there must be access
via "sudo" to root from this login. Privilege is required to override
configuration change checks and for things like cleaning up spool files,
but on the other hand, the tests themselves need to call Exim from a
non-root process. The use of "sudo" is the easiest way to achieve all this.
The test script uses "sudo" to do a number of things as root, so it is best
if you set a sudo timeout so that you do not have to keep typing a
password. For example, if you put
Defaults timestamp_timeout=480
in /etc/sudoers, a password lasts for 8 hours (a working day). It is
probably not a good idea to run the tests as the Exim user, as this is
recognized as special by Exim.
(3) The login under which you run the tests must be in the exim group so that
it has access to logs, spool files, etc. The login should not be one of the
names "userx", "usery", "userz", or a few other simple ones such as "abcd"
and "xyz" and single letters that are used in the tests. (The original
tests use my login a lot; I'm weeding this out as I convert, and I'll try
to get rid of common names as well.) The test suite expects the login to
have a gecos name; I think it will now run if the gecos field is empty but
there may be anomalies.
(4) The directory into which you unpack the test suite must be accessible by
the Exim user, so that code which is running as exim can access the files
therein. A world-readable directory is fine. However, there may be problems
if the path name of the directory is excessively long. This is because it
sometimes appears in logs lines or debug output, and if it is truncated, it
is no longer recognized.
(5) Exim must be built with its user and group specified at build time, and
with certain minimum facilities, namely:
Routers: accept, dnslookup, manualroute, redirect
Transports: appendfile, autoreply, pipe, smtp
Lookups: lsearch
Most Exim binaries will have these included.
(6) A C compiler is needed to build some test programs, and the test script is
written in Perl, so you need that.
(7) Some of the tests run Exim as a daemon, and others use a testing server
(described below). These require TCP ports. In the configurations and
scripts, the ports are parameterized, but at present, fixed values are
written into the controlling script. These are ports 1224 to 1229. If these
ports are not available for use, some of the tests will fail.
(8) There is an underlying assumption that the host on which the tests are
being run has an IPv4 address (which the test script seeks out). If there
is also an IPv6 address, additional tests are run when the Exim binary
contains IPv6 support. There are checks in the scripts for a running IPv4
interface; when one is not found, some tests are skipped (with a warning
message).
OPTIONAL EXTRAS
---------------
If the Exim binary that is being tested contains extra functionality in
addition to the minimum specified above, additional tests are run to exercise
the extra functionality, except for a few special cases such as the databases
(MySQL, PostgreSQL, LDAP) where special data is needed for the tests.
RUNNING THE TEST SUITE
----------------------
(1) Download the tarball exim-testsuite-x.xx.tar.bz2 and unpack it, preferably
in a directory alongside an Exim source directory (see below).
(2) cd into the exim-testsuite-x.xx directory.
(3) Run "./configure" and then "make". This builds a few auxiliary programs
that are written in C.
(4) Run "./runtest" (a Perl script) as described below.
(5) If you want to see what tests are available, run "./listtests".
BREAKING OUT OF THE TEST SCRIPT
-------------------------------
If you abandon the test run by typing ^C, the interrupt may be passed to a
program that the script is running, or it may be passed to the script itself.
In the former case, the script should detect that the program has ended
abnormally. In both cases, the script tries to clean up everything, including
killing any Exim daemons that it has started. However, there may be race
conditions in which the clean up does not happen. If, after breaking out of a
run, you see strange errors in the next run, look for any left-over Exim
daemons, and kill them by hand.
THE LISTTESTS SCRIPT
--------------------
The individual test scripts are in subdirectories of the "scripts" directory.
If you do not supply any arguments to ./listtests, it scans all the scripts in
all the directories, and outputs the heading line from each script. The output
is piped through "less", and begins like this:
=== 0000-Basic ===
Basic/0001 Basic configuration setting
Basic/0002 Common string expansions
Basic/0003 Caseless address blocking
...
Lines that start === give the name of the subdirectory containing the test
scripts that follow. If you supply an argument to ./listtests, it is used as a
Perl pattern to match case-independently against the names of the
subdirectories. Only those that match are scanned. For example, "./listtests
ipv6" outputs this:
=== 1000-Basic-ipv6 ===
=== Requires: support IPv6
Basic-ipv6/1000 -bh and non-canonical IPv6 addresses
Basic-ipv6/1001 recognizing IPv6 address in HELO/EHLO
=== 2250-dnsdb-ipv6 ===
=== Requires: support IPv6
lookup dnsdb
dnsdb-ipv6/2250 dnsdb ipv6 lookup in string expansions
If you supply a second argument to ./listtests, it is used as a Perl pattern to
match case-independently against the individual script titles. For example,
"./listtests . mx" lists all tests whose titles contain "mx", because "."
matches all the subdirectory names.
THE RUNTEST SCRIPT
------------------
If you do not supply any arguments to ./runtest, it searches for an Exim
source tree at the same level as the test suite directory. It then looks for an
Exim binary in a "build" directory of that source tree. If there are several
Exim source trees, it chooses the latest version of Exim. Consider the
following example:
$ ls -F /source/exim
exim-4.50/ exim-4.52/ exim-testsuite-0.00/
A simple ./runtest from within the test suite will use a 4.52 binary if it
finds one, otherwise a 4.50 binary. If a binary cannot be found, the script
prompts for one. Alternatively, you can supply the binary on the command line:
./runtest /usr/exim/bin/exim
The test suite also uses some of the Exim utilities (such as exim_dbmbuild),
and it expects to find them in the same directory as Exim itself. If they are
not found, the tests that use them are omitted. A suitable comment is output.
On the ./runtest command line, following the name of the binary, if present,
there may be a number of options and then one or two numbers. The full syntax
is as follows:
./runtest [binary name] [runtest options] [exim options] \
[first test] [last test]
There are some options for the ./runtest script itself:
-DEBUG This option is for debugging the test script. It causes some
tracing information to be output.
-DIFF By default, file comparisons are done using a private compare
command called "cf", which is built from source that is provided in
the src directory. This is a command I've had for nearly 20 years -
look at the source comments for its history - whose output I
prefer. However, if you want to use "diff" instead, give -DIFF as a
runtest option. In that case, "diff -u" is used for comparisons.
(If it turns out that most people prefer to use diff, I'll change
the default.)
-KEEP Normally, after a successful run, the test output files are
deleted. This option prevents this. It is useful when running a
single test, in order to look at the actual output before it is
modified for comparison with saved output.
-NOIPV4 Pretend that an IPv4 interface was not found. This is useful for
testing that the test suite correctly skips tests that require
a running IPv4 interface.
-NOIPV6 Pretend that an IPv6 interface was not found. This is useful for
testing that the test suite correctly skips tests that require
a running IPv6 interface.
-UPDATE If this option is set, any detected changes in test output are
automatically accepted and used to update the stored copies of the
output. It is a dangerous option, but it useful for the test suite
maintainer after making a change to the code that affects a lot of
tests (for example, the wording of a message).
The options for ./runtest must be given first (but after the name of the
binary, if present). Any further options, that is, items on the command line
that start with a hyphen, are passed to the Exim binary when it is run as part
of a test. The only sensible use of this is to pass "-d" in order to run a test
with debugging enabled. Any other options are likely to conflict with options
that are set in the tests. Some tests are already set up to run with debugging.
In these cases, -d on the command line overrides their own debug settings.
The final two arguments specify the range of tests to be run. Test numbers lie
in the range 1 to 9999. If no numbers are given, the defaults are 1 and 8999
(sic). Tests with higher numbers (9000 upwards) are not run automatically
because they require specific data (such as a particular MySQL table) that is
unlikely to be generally available.
Tests that require certain optional features of Exim are grouped by number, so
in any given range, not all the tests will exist. Non-existent tests are just
skipped, but if there are no tests at all in the given range, a message is
output.
If you give only one number, just that test is run (if it exists). Instead of a
second number, you can give the character "+", which is interpreted as "to the
end". Normally this is 8999; if the starting number is 9000 or higher, "+" is
interpreted as 9999. Examples:
./runtest 1300
./runtest 1400 1699
./runtest /usr/sbin/exim 5000 +
./runtest -DIFF -d 81
When the script starts up, the first thing it does is to check that you have
sudo access to root. Then it outputs the version number of the Exim binary that
it is testing, and also information about the optional facilities that are
present (obtained from "exim -bV"). This is followed by some environmental
information, including the current login id and the hosts's IP address. The
script checks that the current user is in the Exim group, and that the Exim
user has access to the test suite directory.
The script outputs the list of tests requested, and a list of tests that will
be omitted because the relevant optional facilities are not in the binary. You
are then invited to press Return to start the tests running.
TEST OUTPUT
-----------
When all goes well, the only permanent output is the identity of the tests as
they are run, and "Script completed" for each test script, for example:
Basic/0001 Basic configuration setting
Script completed
Basic/0002 Basic string expansions
Script completed
Basic/0003 Caseless address blocking
Script completed
Basic/0004 Caseful address blocking
Script completed
Basic/0005 -bs to simple local delivery
...
While a script is running, it shows "Test n" on the screen, for each of the
Exim tests within the script. There may also be comments from some tests when a
delay is expected, for example, if there is a "sleep" while testing a timeout.
Before each set of optional tests, an extra identifying line is output. For
example:
>>> The following tests require: authenticator cram_md5
CRAM-MD5/2500 CRAM-MD5 server tests
Script completed
CRAM-MD5/2501 CRAM-MD5 client tests
Script completed
If a test fails, you are shown the output of the text comparison that failed,
and prompted as to what to do next. The output is shown using the "less"
command, or "more" if "less" is not available. By default, the output is from
the "cf" program, and might look like this:
DBM/1300 DBM files and exim_dbmbuild
===============
Lines 7-9 of "test-stdout-munged" do not match lines 7-11 of "stdout/1300".
----------
exim_dbmbuild exit code = 1
Continued set of lines is too long: max permitted length is 99999
exim_dbmbuild exit code = 1
----------
dbmbuild abandoned
exim_dbmbuild exit code = 2
Continued set of lines is too long: max permitted length is 99999
dbmbuild abandoned
exim_dbmbuild exit code = 2
===============
1 difference found.
"test-stdout-munged" contains 16 lines; "stdout/1300" contains 18 lines.
Continue, Update & retry, Quit? [Q]
This example was generated by running the test with a version of Exim
that had a bug in the exim_dbmbuild utility (the bug was fixed at release
4.53). See "How the tests work" below for a description of the files that are
used. In this case, the standard output differed from what was expected.
The reply to the prompt must either be empty, in which case it takes the
default that is given in brackets (in this case Q), or a single letter, in
upper or lower case (in this case, one of C, U, or Q). If you type anything
else, the prompt is repeated.
"Continue" carries on as if the files had matched; that is, it ignores the
mismatch. Any other output files for the same test will be compared before
moving on to the next test.
"Update & retry" copies the new file to the saved file, and reruns the test
after doing any further comparisons that may be necessary.
Other circumstances give rise to other prompts. If a test generates output for
which there is no saved data, the prompt (after a message stating which file is
unexpectely not empty) is:
Continue, Show, or Quit? [Q]
"Show" displays the data on the screen, and then you get the "Continue..."
prompt. If a test ends with an unexpected return code, the prompt is:
show stdErr, show stdOut, Continue (without file comparison), or Quit? [Q]
Typically in these cases there will be something interesting in the stderr
or stdout output. There is a similar prompt after the "server" auxiliary
program fails.
OPENSSL AND GNUTLS ERROR MESSAGES
---------------------------------
Some of the TLS tests deliberately cause errors to check how Exim handles them.
It has been observed that different releases of the OpenSSL and GnuTLS
libraries generate different error messages. This may cause the comparison with
the saved output to fail. Such errors can be ignored.
OTHER SCRIPTS AND PROGRAMS
--------------------------
There is a freestanding Perl script called "listtests" that scans the test
scripts and outputs a list of all the tests, with a short descriptive comment
for each one. Special requirements for groups of tests are also noted.
The main runtest script makes use of a second Perl script and some compiled C
programs. These are:
patchexim A Perl script that makes a patched version of Exim (see the
next section for details).
bin/cf A text comparison program (see above).
bin/checkaccess A program that is run as root; it changes uid/gid to the
Exim user and group, and then checks that it can access
files in the test suite's directory.
bin/client A script-driven SMTP client simulation.
bin/client-gnutls A script-driven SMTP client simulation with GnuTLS support.
This is built only if GnuTLS support is detected on the host.
bin/client-ssl A script-driven SMTP client simulation with OpenSSL support.
This is built only if OpenSSL support is detected on the
host.
bin/fakens A fake "nameserver" for DNS tests (see below for details).
bin/fd A program that outputs details of open file descriptors.
bin/iefbr14 A program that does nothing, and returns 0. It's just like
the "true" command, but it is in a known place.
bin/loaded Some dynamically loaded functions for testing dlfunc support.
bin/server A script-driven SMTP server simulation.
The runtest script also makes use of a number of ordinary commands such as
"cp", "kill", "more", and "rm", via the system() call. In some cases these are
run as root by means of sudo.
STANDARD SUBSTITUTIONS
----------------------
In the following sections, there are several references to the "standard
substitutions". These make changes to some of the stored files when they are
used in a test. To save repetition, the substitutions themselves are documented
here:
CALLER is replaced by the login name of the user running the tests
CALLER_GID is replaced by the caller's group id
CALLER_UID is replaced by the caller's user id
DIR is replaced by the name of the test-suite directory
EXIMGROUP is replaced by the name of the Exim group
EXIMUSER is replaced by the name of the Exim user
HOSTIPV4 is replaced by the local host's IPv4 address
HOSTIPV6 is replaced by the local host's IPv6 address
HOSTNAME is replaced by the local host's name
PORT_D is replaced by a port number for normal daemon use
PORT_N is replaced by a port number that should never respond
PORT_S is replaced by a port number for normal bin/server use
TESTNUM is replaced by the current test number
V4NET is replaced by an IPv4 network number for testing
V6NET is replaced by an IPv6 network number for testing
PORT_D is currently hard-wired to 1225, PORT_N to 1223, and PORT_S to 1224.
V4NET is hardwired to 224 and V6NET to ff00. These networks are used for DNS
testing purposes, and for testing Exim with -bh. The only requirement is that
they are networks that can never be used for an IP address of a real host. I've
chosen two multicast networks for the moment.
If the host has no IPv6 address, "<no IPv6 address found>" is substituted but
that does not matter because no IPv6 tests will be run. A similar substitution
is made if there is no IPv4 address, and again, tests that actually require a
running IPv4 interface should be skipped.
If the host has more than one IPv4 or IPv6 address, the first one that
"ifconfig" lists is used. If the only available address is 127.0.0.1 (or ::1
for IPv6) it is used, but another value is prefered if available.
In situations where a specific test is not being run (for example, when setting
up dynamic data files), TESTNUM is replaced by an empty string, but should not
in fact occur in such files.
HOW THE TESTS WORK
------------------
Each numbered script runs Exim (sometimes several times) with its own Exim
configuration file. The configurations are stored in the "confs" directory,
and before running each test, a copy of the appropriate configuration, with the
standard substitutions, is made in the file test-config. The -C command line
option is used to tell Exim to use this configuration.
The -D option is used to pass the path of the Exim binary to the configuration.
This is not standardly substituted, because there are two possible binaries
that might be used in the same test (one setuid to root, the other to the exim
user). Some tests also make use of -D to vary the configuration for different
calls to the Exim binary.
Normally, of course, Exim gives up root privilege when -C and -D are used by
unprivileged users. We do not want this to happen when running the tests,
because we want to be able to test all aspects of Exim, including receiving
mail from unprivileged users. The way this is handled is as follows:
At the start of the runtest script, the patchexim script is run as root. This
script makes a copy of the Exim binary that is to be tested, patching it as it
does so. (This is a binary patch, not a source patch.) The patch causes the
binary, when run, to "know" that it is running in the test harness. It does not
give up root privilege when -C and -D are used, and in a few places it takes
other special actions, such as delaying when starting a subprocess to allow
debug output from the parent to be written first. If you want to know more,
grep the Exim source files for "running_in_test_harness".
The patched binary is placed in the directory eximdir/exim and given the normal
setuid root privilege. This is, of course, a dangerous binary to have lying
around, especially if there are unprivileged users on the system. To protect
it, the eximdir directory is created with the current user as owner, exim as
the group owner, and with access drwx--x---. Thus, only the user who is running
the tests (who is known to have access to root) and the exim user have access
to the modified Exim binary. When runtest terminates, the patched binary is
removed.
Each set of tests proceeds by interpreting its controlling script. The scripts
are in subdirectories of the "scripts" directory. They are split up according
to the requirements of the tests they contain, with the 0000-Basic directory
containing tests that can always be run. Run the "listtests" script to obtain a
list of tests.
TEST OUTPUT
-----------
Output from script runs is written to the files test-stdout and test-stderr.
When an Exim server is involved, test-stdout-server and test-stderr-server are
used for its output. Before being compared with the saved output, the
non-server and server files are concatenated, so a single saved file contains
both.
A directory called spool is used for Exim's spool files, and for Exim logs.
These locations are specified in every test's configuration file.
When messages are delivered to files, the files are put in the test-mail
directory. Output from comparisons is written to test-cf.
Before comparisons are done, output texts are modified ("munged") to change or
remove parts that are expected to vary from run to run. The modified files all
end with the suffix "-munged". Thus, you will see test-stdout-munged,
test-mainlog-munged, test-mail-munged, and so on. Other files whose names start
with "test-" are created and used by some of the tests.
At the end of a successful test run, the spool directory and all the files
whose names begin with "test-" are removed. If the run ends unsuccessfully
(typically after a "Q" response to a prompt), the spool and test files are left
in existence so that the problem can be investigated.
TEST COMMANDS
-------------
Each test script consists of a list of commands, each optionally preceded by
comments (lines starting with #) and (also optionally) a line containing an
expected return code. Some of the commands are followed by data lines
terminated by a line of four asterisks.
The first line of each script must be a comment that briefly describes the
script. For example:
# -bS Use of HELO/RSET
A line consisting just of digits is interpreted as the expected return code
for the command that follows. The default expectation when no such line exists
is a zero return code. For example, here is a complete test script, containing
just one command:
# -bS Unexpected EOF in headers
1
exim -bS -odi
mail from:<someone@???>
rcpt to:<blackhole@HOSTNAME>
data
from: me
****
The expected return code in this case is 1, and the data lines are passed to
Exim on its standard input. Both the command line and the data lines have the
standard substitions applied to them. Thus, HOSTNAME in the example above will
be replaced by the local host's name. Long commands can be continued over
several lines by using \ as a continuation character. This does *not* apply to
data lines.
Here follows a [currently incomplete] list of supported commands. They can be
divided into two groups:
Commands with no input
----------------------
These commands are not followed by any input data, or by a line of asterisks.
dbmbuild <file1> <file1>
This command runs the exim_dbmbuild utility to build a DBM file. It is used
only when DBM support is available in Exim, and typically follows the use of a
"write" command (see below) that creates the input file.
echo <text>
The text is written to the screen; this is used to output comments from
scripts.
gnutls
This command is present at the start of all but one of the tests that use
GnuTLS. It copies a pre-existing parameter file into the spool directory, so
that Exim does not have to re-create the file each time. The first GnuTLS test
does not do this, in order to test that Exim can create the file (it takes some
time).
killdaemon
This command must be given in any script that starts an Exim daemon, normally
at the end. It searches for the PID file in the spool directory, and sends a
SIGINT signal to the Exim daemon process whose PID it finds. See below for
comments about starting Exim daemons.
millisleep <m>
This command causes the script to sleep for m milliseconds. Nothing is output
to the screen.
need_ipv4
This command must be at the head of a script. If no IPv4 interface has been
found, the entire script is skipped, and a comment is output.
need_ipv6
This command must be at the head of a script. If no IPv6 interface has been
found, the entire script is skipped, and a comment is output.
need_move_frozen_messages
This command must be at the head of a script. If the Exim binary does not have
support for moving frozen messages (which is an optional feature), the entire
script is skipped, and a comment is output.
no_message_check
If this command is encountered anywhere in the script, messages that are
delivered when the script runs are not compared with saved versions.
no_msglog_check
If this command is encountered anywhere in the script, message log files that
are still in existence at the end of the run (for messages that were not
delivered) are not compared with saved versions.
no_stderr_check
If this command is encountered anywhere in the script, the stderr output from
the run is not compared with a saved version.
no_stdout_check
If this command is encountered anywhere in the script, the stdout output from
the run is not compared with a saved version.
rmfiltertest
This command indicates that the script is for a certain type of filter test, in
which there are a lot of repetitive stdout lines that get in the way, because
filter tests output data about the sender and recipient. Such lines are removed
from the stdout output before comparing, for ease of human perusal.
sleep <n>
This command causes the script to sleep for n seconds. If n is greater than
one, "sleep <n>" is output to the screen, followed by a dot for every second
that passes.
sortlog
This command causes special sorting to occur on the mainlog file before
comparison. Every sequence of contiguous delivery lines (lines containing the
=> -> or *> flags) is sorted. This is necessary in some tests that use parallel
deliveries because on different systems the processes may terminate in a
different order.
A number of standard file management commands are recognized. These are chmod,
chown, ln, ls, du, mkdir, mkfifo, and touch. Some are run as root using "sudo".
Commands with input
-------------------
The remaining commands are followed by data lines for their standard input,
terminated by four asterisks. Even if no data is required for the particular
usage, the asterisks must be given.
catwrite <file name> [nxm[=start-of-line-text]]*
This command operates like the "write" command, which is described below,
except that the out it generates is copied to the end of the test-stdout file
as well as to the named file.
client [<options>] <ip address> <port> [<outgoing interface>]
This command runs the auxiliary "client" program that simulates an SMTP client.
It is controlled by a script read from its standard input, details of which are
given below. The only option is -t, which must be followed by a number, to
specify the command timeout in seconds. The program connects to the given IP
address and port, using the specified interface, if one is given.
client-ssl [<options>] <ip address> <port> [<outgoing interface>] \
[<cert file>] [<key file>]
When OpenSSL is available on the host, an alternative version of the client
program is compiled, one that supports TLS using OpenSSL. The additional
arguments specify a certificate and key file when required. There is one
additional option, -tls-on-connect, that causes the client to initiate TLS
negotiation immediately on connection.
client-gnutls [<options>] <ip address> <port> [<outgoing interface>] \
[<cert file>] [<key file>]
When GnuTLS is available on the host, an alternative version of the client
program is compiled, one that supports TLS using GnuTLS. The additional
arguments specify a certificate and key file when required. There is one
additional option, -tls-on-connect, that causes the client to initiate TLS
negotiation immediately on connection.
exim [<options>] [<arguments>]
This command runs the testing version of Exim. Any occurrence of "$msg1" in the
command line is replaced by the ID of the first (oldest) message in Exim's
(testing) spool. "$msg2" refers to the second, and so on. The name "exim" can
be preceded by an environment setting as in this example:
LDAPTLS_REQCERT=never exim -be
It can also be preceded by a number; this specifies a number of seconds to wait
before closing the stdout pipe to Exim, and is used for some timeout tests. For
example:
3 exim -bs
Finally, "exim" can be preceded by "sudo", to run Exim as root. If more than
one of these prefixes is present, they must be in the above order.
exim_exim [<options>] [<arguments>]
This runs an alternative version of Exim that is setuid to exim rather than to
root.
server [<options>] <port or socket> [<connection count>]
This command runs the auxiliary "server" program that simulates an SMTP (or
other) server. It is controlled by a script that is read from its standard
input, details of which are given below. A number of options are implemented:
-d causes the server to output debugging information
-t sets a timeout in seconds (default 5) for when the server is
awaiting an incoming connection
-noipv4 causes the server not to set up an IPv4 socket
-noipv6 causes the server not to set up an IPv6 socket
By default, in an IPv6 environment, both kinds of socket are set up. However,
the test script knows which interfaces actually exist on the host, and it adds
-noipv4 or -noipv6 to the server command as required. An error occurs if both
these options are given.
The only required argument is either a port number or the path name of a Unix
domain socket. The port is normally PORT_S, which is changed to an actual
number by the standard substitutions. The optional final argument specifies the
number of different connections to expect (default 1). These must happen
serially (one at a time). There is no support for multiple simultaneous
connections. Here are some example commands:
server PORT_S
server -t 10 PORT_S 3
server /tmp/somesocket
The following lines, up to a line of four asterisks, are the server's
controlling standard input (described below). These lines are read and
remembered; during the following commands, until an "exim" command is reached,
the server is run in parallel.
write <file name> [nxm[=start-of-line-text]]*
The "write" command is a way of creating files of specific sizes for buffering
tests, or containing specific data lines. Being able to do this from within the
script saves holding lots of little test files. The optional argument specifies
n lines of length m. The lines consist of the letter "a". If start of line text
is supplied, it replaces "a"s at the start of each line. Underscores in the
start of line text are turned into spaces. The optional argument may be
repeated. The data lines that follow a "write" command are split into two by a
line of four plus signs. Any above the split are written before the
fixed-length lines, and any below the split are written after. For example:
write test-data 3x30=AB_ 1x50
Pre-data
lines
++++
Post-data
lines
****
This command generates a file containing:
Pre-data
lines
AB aaaaaaaaaaaaaaaaaaaaaaaaaaa
AB aaaaaaaaaaaaaaaaaaaaaaaaaaa
AB aaaaaaaaaaaaaaaaaaaaaaaaaaa
aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
Post-data
lines
If there are no fixed-length line specifiers, there is no need to split the
data, and a line of plusses is not needed.
[sudo] perl
This command runs Perl, with the data as its standard input, to allow arbitrary
one-off things to be done.
CLIENT SCRIPTS
--------------
Lines in client scripts are of two kinds:
(1) If a line begins with three question marks and a space, the rest of the
line defines the start of expected output from the server. If what is
received does not match, the client bombs out with an error message.
(2) If a line starts with three plus signs followed by a space, the rest of the
line specifies a number of seconds to sleep for before proceeding.
(3) Otherwise, the line is an input line line that is sent to the server. Any
occurrences of \r and \n in the line are turned into carriage return and
linefeed, respectively. This is used for testing PIPELINING.
Here is a simple example:
client 127.0.0.1 PORT_D
??? 250
EHLO xxx
??? 250-
??? 250
AUTH PLAIN AbdXi0AdnD2CVy
??? 535
quit
??? 221
****
In the case of client-gnutls and client-ssl, if a command is "starttls", this
is remembered, and after a subsequent OK response, an attempt to move into TLS
mode occurs. If a command is "starttls_wait", the client sends "starttls" but
does not start up TLS; this is for testing timeouts. If a command is "stoptls",
an existing TLS connection is shut down, but nothing is sent.
SERVER SCRIPTS
--------------
The server program sleeps till a connection occurs or its timeout is reached,
in which case it bombs out. The next set of command lines are interpreted. They
are of the following kinds:
(1) A line that starts with '>' or with a digit is an output line that is sent
to the client. In the case of '>':
(a) If the line starts with ">>", no terminating CRLF is sent.
(b) If the line starts with ">CR>", just CR is sent at the end.
(c) If the line starts with ">LF>", just LF is sent at the end.
(d) If the line starts with ">*eof", nothing is sent and the connection
is closed.
The data that is sent starts after the initial '>' sequence.
(2) A line that starts with "*sleep" specifies a number of seconds to wait
before proceeding.
(3) A line containing "*eof" specifies that the client is expected to close
the connection at this point.
(4) A line containing just '.' specifies that the client is expected to send
many lines, terminated by one that contains just a dot.
(5) Otherwise, the line defines the start of an input line that the client
is expected to send. To allow for lines that start with digits, the line
may start with '<', which is not taken as part of the input data. If the
input does not match, the server bombs out with an error message.
Here is a simple server example:
server PORT_S
220 Greetings
EHLO
250 Hello there
MAIL FROM
250 OK
RCPT TO
250 OK
DATA
354 Send it!
.
250 OK
QUIT
225 OK
****
After a "server" command in a test script, the server runs in parallel until an
"exim" command is reached. The "exim" command attempts to deliver one or more
messages to port PORT_S on the local host. When it has finished, the test
script waits for the "server" process to finish.
AUXILIARY DATA FILES
--------------------
Many of the tests make use of auxiliary data files. There are two types; those
whose content is fixed, and those whose content needs to be varied according to
the current environment. The former are kept in the directory aux-fixed. The
latter are distributed in the directory aux-var-src, and copied with the
standard substitutions into the directory aux-var at the start of each test
run.
Most of the auxiliary files have names that start with a test number,
indicating that they are specific to that one test. A few fixed files (for
example, some TLS certificates) are used by more than one test, and so their
names are not of this form.
There are also some auxilary DNS zone files, which are described in the next
section.
DNS LOOKUPS AND GETHOSTBYNAME
-----------------------------
The original test suite required special testing zones to be loaded into a
local nameserver. This is no longer a requirement for the new suite. Instead, a
program called fakens is used to simulate a nameserver. When Exim is running in
the test harness, instead of calling res_search() - the normal call to the DNS
resolver - it calls a testing function. This handles a few special names itself
(for compatibility with the old test suite), but otherwise passes the query to
the fakens program.
The fakens program consults "zone files" in the directory called dnszones, and
returns data in the standard resource record format for Exim to process as if
it came from the DNS. However, if the requested domain is not in any of the
zones that fakens knows about, it returns a special code that causes Exim to
pass the query on to res_search(). The zone files are:
db.test.ex A zone for the domain test.ex.
db.ip4.10 A zone for one special case in 10.250.0.0/16 (see below)
db.ip4.V4NET A zone for the domain V4NET.in-addr.arpa.
db.ip4.127 A zone for the domain 127.in-addr.arpa.
db.ip6.V6NET A zone for the domain inverted(V6NET).ip6.arpa.
db.ip6.0 A zone for the domain 0.ip6.arpa.
V4NET and V6NET are substituted with the current testing networks (see above).
In the case of V6NET, the network is four hex digits, and it is split and
inverted appropriately when setting up the zone.
These fake zone files are built dynamically from sources in the dnszones-src
directory by applying the standard substitutions. The test suite also builds
dynamic zone files for the name of the current host and its IP address(es). The
idea is that there should not be any need to rely on an external DNS.
The domain names that are handled directly by Exim, without being passed to
fakens, are:
test.again.dns This always provokes a TRY_AGAIN response, for testing the
handling of temporary DNS error. If the full domain name
starts with digits, a delay of that many seconds occurs.
test.fail.dns This always provokes a NO_RECOVERY response, for testing
DNS server failures.
This special handling could now be done in the fakens program, but while the
old test suite is still being used it has to be done in Exim itself, so for the
moment it remains there.
The use of gethostbyname() and its IPv6 friends is also subverted when Exim is
running in the test harness. The test code handles a few special names
directly; for all the others it uses DNS lookups, which are then handled as
just described. Thus, the use of /etc/hosts is completely bypassed. The names
that are specially handled are:
manyhome.test.ex This name is used for testing hosts with ridiculously large
numbers of IP addresses; 2048 IP addresses are generated
and returned. Doing it this way saves having to make the
interface to fakens handle more records that can fit in the
data block. The addresses that are generated are in the
10.250.0.0/16 network.
localhost Always returns 127.0.0.1 or ::1, for IPv4 and IPv6 lookups,
respectively.
<an IP address> If the IP address is of the correct form for the lookup
type (IPv4 or IPv6), it is returned. Otherwise a panic-die
error occurs.
The reverse zone db.ip4.10 is provided just for the manyhome.test.ex case. It
contains a single wildcard resource record. It also contains the line
PASS ON NOT FOUND
Whenever fakens finds this line in a zone file, it returns PASS_ON instead of
HOST_NOT_FOUND. This causes Exim to pass the query to res_search().
****
Index: configure.ac
====================================================================
dnl $Cambridge: exim/exim-test/configure.ac,v 1.1 2006/02/06 16:07:10 ph10 Exp $
dnl Process this file with autoconf to produce a configure script.
dnl This is required at the start; the name is the name of a file
dnl it should be seeing, to verify it is in the same directory.
AC_INIT(listtests)
dnl A safety precaution
AC_PREREQ(2.57)
dnl Checks for programs.
AC_PROG_CC
dnl Checks for header files.
AC_CHECK_HEADERS(openssl/crypto.h,[CLIENT_SSL=bin/client-ssl])
AC_CHECK_HEADERS(gnutls/gnutls.h,[CLIENT_GNUTLS=bin/client-gnutls])
dnl The check on dynamically loaded modules requires the building of
dnl something to load. This seems to be something that varies between
dnl systems and compilers something awful. Therefore, we enable it only
dnl for those systems and compilers that we know about.
dnl I tried using AC_CANONICAL_HOST, but it insisted on looking for an
dnl "install" script for some weird reason.
host_os=`uname -s`
case $CC-$host_os in
gcc-*linux* | gcc-*Linux* | gcc-*LINUX* | gcc-FreeBSD)
LOADED=bin/loaded
LOADED_OPT=-shared
echo "Using gcc on $host_os: will compile dynamically loaded module"
;;
*)
LOADED=
echo "Will not compile dynamically loaded module: not known OS/CC combination"
;;
esac
dnl "Export" these variables
AC_SUBST(CLIENT_SSL)
AC_SUBST(CLIENT_GNUTLS)
AC_SUBST(LOADED)
AC_SUBST(LOADED_OPT)
dnl This must be last; it determines what files are written
AC_OUTPUT(Makefile)
Index: listtests
====================================================================
#! /bin/sh
# $Cambridge: exim/exim-test/listtests,v 1.1 2006/02/06 16:07:10 ph10 Exp $
# This script scans the directories of Exim test scripts and lists the first
# comment line of each one, which gives a description. The output is piped via
# "more". If the script has an argument, it is a pattern that is used to select
# only certain subdirectories. If the script has a second argument, it is a
# pattern that is used to select only certain test titles from the selected
# directories.
/usr/bin/perl -w - "$1" "$2" <<'PerlEnd' | less
$dirpat = "$ARGV[0]";
$filpat = "$ARGV[1]";
opendir(SCRIPTS, "scripts") || die "** Failed to opendir(SCRIPTS): $!\n";
@subdirs = readdir(SCRIPTS);
closedir(SCRIPTS);
foreach $subdir (sort @subdirs)
{
my($first) = 1;
next if $subdir =~ /^\./;
next if $dirpat ne "" && $subdir !~ /$dirpat/i;
opendir(TESTS, "scripts/$subdir") ||
die "** Failed to opendir(scripts/$subdir): $!\n";
@tests = readdir(TESTS);
closedir(TESTS);
foreach $file (sort @tests)
{
next if $file !~ /^\d\d\d\d$/;
open(IN, "scripts/$subdir/$file") ||
die "** Failed to open scripts/$subdir/$file: $!\n";
my($heading) = substr(<IN>, 2);
close(IN);
if ($filpat eq "" || $heading =~ /$filpat/i)
{
if ($first)
{
print "\n=== $subdir ===\n";
if (open(REQUIRES, "scripts/$subdir/REQUIRES"))
{
my($indent) = "";
print "=== Requires: ";
while (<REQUIRES>)
{
print $indent, $_;
$indent = " ";
}
print "\n" if $indent eq "";
close (REQUIRES);
}
$first = 0;
}
printf("%s/%s %s", (substr $subdir, 5), $file, $heading);
}
}
}
PerlEnd
# End
Index: patchexim
====================================================================
#! /usr/bin/perl -w
# $Cambridge: exim/exim-test/patchexim,v 1.1 2006/02/06 16:07:10 ph10 Exp $
###############################################################################
# This is an auxiliary script that is part of the Exim test suite. It must be #
# run as root, and is normally called from the main controlling script. Its #
# job is to make a copy of Exim, suitably patched so that it can run in the #
# test harness. See further comments in the main script. #
# #
# The only argument to this script is the name of the Exim binary that is to #
# be copied. The script must be run in the correct current directory. #
###############################################################################
open(IN, "$ARGV[0]") || die "** Failed to open $ARGV[0]: $!\n";
open(OUT, ">eximdir/exim") || die "** Failed to open eximdir/exim: $!\n";
while(<IN>)
{
s/>>>running<<</<<<testing>>>/;
s/(\d+\.\d+(?:\.\d+)?(-RC\d+)?\0<<eximversion>>)/"x.yz\0" . ("*" x (length($1) - 5))/e;
print OUT;
}
close(IN);
close(OUT);
chmod 04755, "eximdir/exim";
# End of patchexim script
Index: runtest
====================================================================
#! /usr/bin/perl -w
# $Cambridge: exim/exim-test/runtest,v 1.1 2006/02/06 16:07:10 ph10 Exp $
###############################################################################
# This is the controlling script for the "new" test suite for Exim. It should #
# be possible to export this suite for running on a wide variety of hosts, in #
# contrast to the old suite, which was very dependent on the environment of #
# Philip Hazel's desktop computer. This implementation inspects the version #
# of Exim that it finds, and tests only those features that are included. The #
# surrounding environment is also tested to discover what is available. See #
# the README file for details of how it all works. #
# #
# Implementation started: 03 August 2005 by Philip Hazel #
# Placed in the Exim CVS: 06 February 2006 #
###############################################################################
require Cwd;
use Errno;
use FileHandle;
use Socket;
# Start by initializing some global variables
$testversion = "4.61 (06-Feb-06)";
$cf = "bin/cf";
$cr = "\r";
$debug = 0;
$force_update = 0;
$more = "less -XF";
$optargs = "";
$save_output = 0;
$server_opts = "";
$have_ipv4 = 1;
$have_ipv6 = 1;
$test_start = 1;
$test_end = $test_top = 8999;
$test_special_top = 9999;
@test_list = ();
@test_dirs = ();
# Networks to use for DNS tests. We need to choose some networks that will
# never be used so that there is no chance that the host on which we are
# running is actually in one of the test networks. Private networks such as
# the IPv4 10.0.0.0/8 network are no good because hosts may well use them.
# Rather than use some unassigned numbers (that might become assigned later),
# I have chosen some multicast networks, in the belief that such addresses
# won't ever be assigned to hosts. This is the only place where these numbers
# are defined, so it is trivially possible to change them should that ever
# become necessary.
$parm_ipv4_test_net = "224";
$parm_ipv6_test_net = "ff00";
# Port numbers are currently hard-wired
$parm_port_n = 1223; # Nothing listening on this port
$parm_port_s = 1224; # Used for the "server" command
$parm_port_d = 1225; # Used for the Exim daemon
$parm_port_d2 = 1226; # Additional for daemon
$parm_port_d3 = 1227; # Additional for daemon
$parm_port_d4 = 1228; # Additional for daemon
###############################################################################
###############################################################################
# Define a number of subroutines
###############################################################################
###############################################################################
##################################################
# Handle signals #
##################################################
sub pipehandler { $sigpipehappened = 1; }
sub inthandler { print "\n"; tests_exit(-1, "Caught SIGINT"); }
##################################################
# Do global macro substitutions #
##################################################
# This function is applied to configurations, command lines and data lines in
# scripts, and to lines in the files of the aux-var-src and the dnszones-src
# directory. It takes one argument: the current test number, or zero when
# setting up files before running any tests.
sub do_substitute{
s?\bCALLER\b?$parm_caller?g;
s?\bCALLER_UID\b?$parm_caller_uid?g;
s?\bCALLER_GID\b?$parm_caller_gid?g;
s?\bCLAMSOCKET\b?$parm_clamsocket?g;
s?\bDIR/?$parm_cwd/?g;
s?\bEXIMGROUP\b?$parm_eximgroup?g;
s?\bEXIMUSER\b?$parm_eximuser?g;
s?\bHOSTIPV4\b?$parm_ipv4?g;
s?\bHOSTIPV6\b?$parm_ipv6?g;
s?\bHOSTNAME\b?$parm_hostname?g;
s?\bPORT_D\b?$parm_port_d?g;
s?\bPORT_D2\b?$parm_port_d2?g;
s?\bPORT_D3\b?$parm_port_d3?g;
s?\bPORT_D4\b?$parm_port_d4?g;
s?\bPORT_N\b?$parm_port_n?g;
s?\bPORT_S\b?$parm_port_s?g;
s?\bTESTNUM\b?$_[0]?g;
s?(\b|_)V4NET([\._])?$1$parm_ipv4_test_net$2?g;
s?\bV6NET:?$parm_ipv6_test_net:?g;
}
##################################################
# Subroutine to tidy up and exit #
##################################################
# In all cases, we check for any Exim daemons that have been left running, and
# kill them. Then remove all the spool data, test output, and the modified Exim
# binary if we are ending normally.
# Arguments:
# $_[0] = 0 for a normal exit; full cleanup done
# $_[0] > 0 for an error exit; no files cleaned up
# $_[0] < 0 for a "die" exit; $_[1] contains a message
sub tests_exit{
my($rc) = $_[0];
my($spool);
# Search for daemon pid files and kill the daemons. We kill with SIGINT rather
# than SIGTERM to stop it outputting "Terminated" to the terminal when not in
# the background.
if (opendir(DIR, "spool"))
{
my(@spools) = sort readdir(DIR);
closedir(DIR);
foreach $spool (@spools)
{
next if $spool !~ /^exim-daemon./;
open(PID, "spool/$spool") || die "** Failed to open \"spool/$spool\": $!\n";
chomp($pid = <PID>);
close(PID);
print "Tidyup: killing daemon pid=$pid\n";
system("sudo rm -f spool/$spool; sudo kill -SIGINT $pid");
}
}
else
{ die "** Failed to opendir(\"spool\"): $!\n" unless $!{ENOENT}; }
# Close the terminal input and remove the test files if all went well, unless
# the option to save them is set. Always remove the patched Exim binary. Then
# exit normally, or die.
close(T);
system("sudo /bin/rm -rf ./spool test-* ./dnszones/*")
if ($rc == 0 && !$save_output);
system("sudo /bin/rm -rf ./eximdir/*");
exit $rc if ($rc >= 0);
die "** runtest error: $_[1]\n";
}
##################################################
# Subroutines used by the munging subroutine #
##################################################
# This function is used for things like message ids, where we want to generate
# more than one value, but keep a consistent mapping throughout.
#
# Arguments:
# $oldid the value from the file
# $base a base string into which we insert a sequence
# $sequence the address of the current sequence counter
sub new_value {
my($oldid, $base, $sequence) = @_;
my($newid) = $cache{$oldid};
if (! defined $newid)
{
$newid = sprintf($base, $$sequence++);
$cache{$oldid} = $newid;
}
return $newid;
}
# This is used while munging the output from exim_dumpdb. We cheat by assuming
# that the date always the same, and just return the number of seconds since
# midnight.
sub date_seconds {
my($day,$month,$year,$hour,$min,$sec) =
$_[0] =~ /^(\d\d)-(\w\w\w)-(\d{4})\s(\d\d):(\d\d):(\d\d)/;
return $hour * 60 * 60 + $min * 60 + $sec;
}
# This is a subroutine to sort maildir files into time-order. The second field
# is the microsecond field, and may vary in length, so must be compared
# numerically.
sub maildirsort {
return $a cmp $b if ($a !~ /^\d+\.H\d/ || $b !~ /^\d+\.H\d/);
my($x1,$y1) = $a =~ /^(\d+)\.H(\d+)/;
my($x2,$y2) = $b =~ /^(\d+)\.H(\d+)/;
return ($x1 != $x2)? ($x1 <=> $x2) : ($y1 <=> $y2);
}
##################################################
# Subroutine list files below a directory #
##################################################
# This is used to build up a list of expected mail files below a certain path
# in the directory tree. It has to be recursive in order to deal with multiple
# maildir mailboxes.
sub list_files_below {
my($dir) = $_[0];
my(@yield) = ();
my(@sublist, $file);
opendir(DIR, $dir) || tests_exit(-1, "Failed to open $dir: $!");
@sublist = sort maildirsort readdir(DIR);
closedir(DIR);
foreach $file (@sublist)
{
next if $file eq "." || $file eq ".." || $file eq "CVS";
if (-d "$dir/$file")
{ @yield = (@yield, list_files_below("$dir/$file")); }
else
{ push @yield, "$dir/$file"; }
}
return @yield;
}
##################################################
# Munge a file before comparing #
##################################################
# The pre-processing turns all dates, times, Exim versions, message ids, and so
# on into standard values, so that the compare works. Perl's substitution with
# an expression provides a neat way to do some of these changes.
# We keep a global associative array for repeatedly turning the same values
# into the same standard values throughout the data from a single test.
# Message ids get this treatment (can't be made reliable for times), and
# times in dumped retry databases are also handled in a special way, as are
# incoming port numbers.
# On entry to the subroutine, the file to write to is already opened with the
# name MUNGED. The input file name is the only argument to the subroutine.
# Certain actions are taken only when the name contains "stderr", "stdout",
# or "log". The yield of the function is 1 if a line matching "*** truncated
# ***" is encountered; otherwise it is 0.
sub munge {
my($file) = $_[0];
my($yield) = 0;
my(@saved) = ();
open(IN, "$file") || tests_exit(-1, "Failed to open $file: $!");
my($is_log) = $file =~ /log/;
my($is_stdout) = $file =~ /stdout/;
my($is_stderr) = $file =~ /stderr/;
# Date pattern
$date = "\\d{2}-\\w{3}-\\d{4}\\s\\d{2}:\\d{2}:\\d{2}";
# Pattern for matching pids at start of stderr lines; initially something
# that won't match.
$spid = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
# Scan the file and make the changes. Near the bottom there are some changes
# that are specific to certain file types, though there are also some of those
# inline too.
while(<IN>)
{
# Check for "*** truncated ***"
$yield = 1 if /\*\*\* truncated \*\*\*/;
# Replace the name of this host
s/\Q$parm_hostname\E/the.local.host.name/g;
# But convert "name=the.local.host address=127.0.0.1" to use "localhost"
s/name=the\.local\.host address=127\.0\.0\.1/name=localhost address=127.0.0.1/g;
# Replace the path to the testsuite directory
s?\Q$parm_cwd\E?TESTSUITE?g;
# Replace the Exim version number (may appear in various places)
s/Exim \d+\.\d+[\w-]*/Exim x.yz/i;
# Replace Exim message ids by a unique series
s/((?:[^\W_]{6}-){2}[^\W_]{2})
/new_value($1, "10Hm%s-0005vi-00", \$next_msgid)/egx;
# The names of lock files appear in some error and debug messages
s/\.lock(\.[-\w]+)+(\.[\da-f]+){2}/.lock.test.ex.dddddddd.pppppppp/;
# Unless we are in an IPv6 test, replace IPv4 and/or IPv6 in "listening on
# port" message, because it is not always the same.
s/port (\d+) \([^)]+\)/port $1/g
if !$is_ipv6test && m/listening for SMTP(S?) on port/;
# Challenges in SPA authentication
s/TlRMTVNTUAACAAAAAAAAAAAoAAABgg[\w+\/]+/TlRMTVNTUAACAAAAAAAAAAAoAAABggAAAEbBRwqFwwIAAAAAAAAAAAAt1sgAAAAA/;
# PRVS values
s?prvs=([^/]+)/[\da-f]{10}@?prvs=$1/xxxxxxxxxx@?g;
# Error lines on stdout from SSL contain process id values and file names.
# They also contain a source file name and line number, which may vary from
# release to release.
s/^\d+:error:/pppp:error:/;
s/:(?:\/[^\s:]+\/)?([^\/\s]+\.c):\d+:/:$1:dddd:/;
# One error test in expansions mentions base 62 or 36
s/is not a base (36|62) number/is not a base 36\/62 number/;
# This message sometimes has a different number of seconds
s/forced fail after \d seconds/forced fail after d seconds/;
# This message may contain a different DBM library name
s/Failed to open \S+( \([^\)]+\))? file/Failed to open DBM file/;
# The message for a non-listening FIFO varies
s/:[^:]+: while opening named pipe/: Error: while opening named pipe/;
# The name of the shell may vary
s/\s\Q$parm_shell\E\b/ SHELL/;
# Debugging output of lists of hosts may have different sort keys
s/sort=\S+/sort=xx/ if /^\S+ (?:\d+\.){3}\d+ mx=\S+ sort=\S+/;
# Random local part in callout cache testing
s/myhost.test.ex-\d+-testing/myhost.test.ex-dddddddd-testing/;
# ======== Dumpdb output ========
# This must be before the general date/date munging.
# Time data lines, which look like this:
# 25-Aug-2000 12:11:37 25-Aug-2000 12:11:37 26-Aug-2000 12:11:37
if (/^($date)\s+($date)\s+($date)(\s+\*)?\s*$/)
{
my($date1,$date2,$date3,$expired) = ($1,$2,$3,$4);
$expired = "" if !defined $expired;
my($increment) = date_seconds($date3) - date_seconds($date2);
# We used to use globally unique replacement values, but timing
# differences make this impossible. Just show the increment on the
# last one.
printf MUNGED ("first failed = time last try = time2 next try = time2 + %s%s\n",
$increment, $expired);
next;
}
# more_errno values in exim_dumpdb output which are times
s/T:(\S+)\s-22\s(\S+)\s/T:$1 -22 xxxx /;
# ======== Dates and times ========
# Dates and times are all turned into the same value - trying to turn
# them into different ones cannot be done repeatedly because they are
# real time stamps generated while running the test. The actual date and
# time used was fixed when I first started running automatic Exim tests.
# Date/time in header lines and SMTP responses
s/[A-Z][a-z]{2},\s\d\d?\s[A-Z][a-z]{2}\s\d\d\d\d\s\d\d\:\d\d:\d\d\s[-+]\d{4}
/Tue, 2 Mar 1999 09:44:33 +0000/gx;
# Date/time in logs and in one instance of a filter test
s/^\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d(\s[+-]\d\d\d\d)?/1999-03-02 09:44:33/gx;
s/^Logwrite\s"\d{4}-\d\d-\d\d\s\d\d:\d\d:\d\d/Logwrite "1999-03-02 09:44:33/gx;
# Date/time in message separators
s/(?:[A-Z][a-z]{2}\s){2}\d\d\s\d\d:\d\d:\d\d\s\d\d\d\d
/Tue Mar 02 09:44:33 1999/gx;
# Date of message arrival in spool file as shown by -Mvh
s/^\d{9,10}\s0$/ddddddddd 0/;
# Date/time in mbx mailbox files
s/\d\d-\w\w\w-\d\d\d\d\s\d\d:\d\d:\d\d\s[-+]\d\d\d\d,/06-Sep-1999 15:52:48 +0100,/gx;
# Date/time in debugging output for writing retry records
if (/^ first failed=(\d+) last try=(\d+) next try=(\d+) (.*)$/)
{
my($next) = $3 - $2;
$_ = " first failed=dddd last try=dddd next try=+$next $4\n";
}
# Time to retry may vary
s/time to retry = -\d+/time to retry = -ddddd/;
s/retry record exists: age=\d/retry record exists: age=d/;
# Date/time in exim -bV output
s/\d\d-[A-Z][a-z]{2}-\d{4}\s\d\d:\d\d:\d\d/07-Mar-2000 12:21:52/g;
# ======== Caller's login, uid, gid, home ========
s/\Q$parm_caller_home\E/CALLER_HOME/g; # NOTE: these must be done
s/\b\Q$parm_caller\E\b/CALLER/g; # in this order!
s/\b\Q$parm_caller_group\E\b/CALLER/g; # In case group name different
s/\beuid=$parm_caller_uid\b/euid=CALLER_UID/g;
s/\begid=$parm_caller_gid\b/egid=CALLER_GID/g;
s/\buid=$parm_caller_uid\b/uid=CALLER_UID/g;
s/\bgid=$parm_caller_gid\b/gid=CALLER_GID/g;
# When looking at spool files with -Mvh, we will find not only the caller
# login, but also the uid and gid. It seems that $) in some Perls gives all
# the auxiliary gids as well, so don't bother checking for that.
s/^CALLER $> \d+$/CALLER UID GID/;
# There is one case where the caller's login is forced to something else,
# in order to test the processing of logins that contain spaces. Weird what
# some people do, isn't it?
s/^spaced user $> \d+$/CALLER UID GID/;
# ======== Exim's login ========
# For bounce messages, this will appear on the U= lines in logs and also
# after Received: and in addresses. In one pipe test it appears after
# "Running as:". It also appears in addresses, and in the names of lock
# files.
s/U=$parm_eximuser/U=EXIMUSER/;
s/user=$parm_eximuser/user=EXIMUSER/;
s/login=$parm_eximuser/login=EXIMUSER/;
s/Received: from $parm_eximuser /Received: from EXIMUSER /;
s/Running as: $parm_eximuser/Running as: EXIMUSER/;
s/\b$parm_eximuser@/EXIMUSER@/;
s/\b$parm_eximuser\.lock\./EXIMUSER.lock./;
s/\beuid=$parm_exim_uid\b/euid=EXIM_UID/g;
s/\begid=$parm_exim_gid\b/egid=EXIM_GID/g;
s/\buid=$parm_exim_uid\b/uid=EXIM_UID/g;
s/\bgid=$parm_exim_gid\b/gid=EXIM_GID/g;
# ======== General uids, gids, and pids ========
# Note: this must come after munges for caller's and exim's uid/gid
s/\bgid=\d+/gid=gggg/;
s/\begid=\d+/egid=gggg/;
s/\bpid=\d+/pid=pppp/;
s/\buid=\d+/uid=uuuu/;
s/\beuid=\d+/euid=uuuu/;
s/set_process_info:\s+\d+/set_process_info: pppp/;
s/queue run pid \d+/queue run pid ppppp/;
s/process \d+ running as transport filter/process pppp running as transport filter/;
s/process \d+ writing to transport filter/process pppp writing to transport filter/;
s/reading pipe for subprocess \d+/reading pipe for subprocess pppp/;
s/remote delivery process \d+ ended/remote delivery process pppp ended/;
# Pid in temp file in appendfile transport
s"test-mail/temp\.\d+\."test-mail/temp.pppp.";
# Detect a daemon stderr line with a pid and save the pid for subsequent
# removal from following lines.
$spid = $1 if /^(\s*\d+) (?:listening|LOG: MAIN|(?:daemon_smtp_port|local_interfaces) overridden by)/;
s/^$spid //;
# Queue runner waiting messages
s/waiting for children of \d+/waiting for children of pppp/;
s/waiting for (\S+) \(\d+\)/waiting for $1 (pppp)/;
# ======== Port numbers ========
# Incoming port numbers may vary, but not in daemon startup line.
s/^Port: (\d+)/"Port: " . new_value($1, "%s", \$next_port)/e;
s/\(port=(\d+)/"(port=" . new_value($1, "%s", \$next_port)/e;
# This handles "connection from" and the like, when the port is given
if (!/listening for SMTP on/ && !/Connecting to/ && !/=>/ && !/\*>/ &&
!/Connection refused/)
{
s/\[([a-z\d:]+|\d+(?:\.\d+){3})\]:(\d+)/"[".$1."]:".new_value($2,"%s",\$next_port)/ie;
}
# Port in host address in spool file output from -Mvh
s/^-host_address (.*)\.\d+/-host_address $1.9999/;
# ======== Local IP addresses ========
# The amount of space between "host" and the address in verification output
# depends on the length of the host name. We therefore reduce it to one space
# for all of them.
s/^\s+host\s(\S+)\s+(\S+)/ host $1 $2/;
s/^\s+(host\s\S+\s\S+)\s+(port=.*)/ host $1 $2/;
s/^\s+(host\s\S+\s\S+)\s+(?=MX=)/ $1 /;
s/host\s\Q$parm_ipv4\E\s\[\Q$parm_ipv4\E\]/host ipv4.ipv4.ipv4.ipv4 [ipv4.ipv4.ipv4.ipv4]/;
s/host\s\Q$parm_ipv6\E\s\[\Q$parm_ipv6\E\]/host ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6 [ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6]/;
s/\b\Q$parm_ipv4\E\b/ip4.ip4.ip4.ip4/g;
s/\b\Q$parm_ipv6\E\b/ip6:ip6:ip6:ip6:ip6:ip6:ip6:ip6/g;
# ======== Test network IP addresses ========
s/(\b|_)\Q$parm_ipv4_test_net\E(?=\.\d+\.\d+\.\d+\b|_|\.rbl|\.in-addr|\.test\.again\.dns)/$1V4NET/g;
s/\b\Q$parm_ipv6_test_net\E(?=:[\da-f]+:[\da-f]+:[\da-f]+)/V6NET/gi;
# ======== IP error numbers and messages ========
# These vary between operating systems
s/Can't assign requested address/Network Error/;
s/Cannot assign requested address/Network Error/;
s/Operation timed out/Connection timed out/;
s/Address family not supported by protocol family/Network Error/;
s/Network is unreachable/Network Error/;
s/Invalid argument/Network Error/;
s/\(\d+\): Network/(dd): Network/;
s/\(\d+\): Connection refused/(dd): Connection refused/;
s/\(\d+\): Connection timed out/(dd): Connection timed out/;
s/\d+ 65 Connection refused/dd 65 Connection refused/;
s/\d+ 321 Connection timed out/dd 321 Connection timed out/;
# ======== Other error numbers ========
s/errno=\d+/errno=dd/g;
# ======== Output from ls ========
# Different operating systems use different spacing on long output
s/ +/ /g if /^[-rwd]{10} /;
# ======== Message sizes =========
# Message sizes vary, owing to different logins and host names that get
# automatically inserted. I can't think of any way of even approximately
# comparing these.
s/([\s,])S=\d+\b/$1S=sss/;
s/:S\d+\b/:Ssss/;
s/^(\s*\d+m\s+)\d+(\s+[a-z0-9-]{16} <)/$1sss$2/i if $is_stdout;
s/\sSIZE=\d+\b/ SIZE=ssss/ if $is_stderr || $is_stdout;
s/\ssize=\d+\b/ size=sss/ if $is_stderr;
s/old size = \d+\b/old size = sssss/;
s/message size = \d+\b/message size = sss/;
s/this message = \d+\b/this message = sss/;
s/Size of headers = \d+/Size of headers = sss/;
s/sum=(?!0)\d+/sum=dddd/;
s/(?<=sum=dddd )count=(?!0)\d+\b/count=dd/;
s/(?<=sum=0 )count=(?!0)\d+\b/count=dd/;
s/,S is \d+\b/,S is ddddd/;
s/\+0100,\d+;/+0100,ddd;/;
s/\(\d+ bytes written\)/(ddd bytes written)/;
s/added '\d+ 1'/added 'ddd 1'/;
# ======== Values in spool space failure message ========
s/space=\d+ inodes=\d+/space=xxxxx inodes=xxxxx/;
# ======== Filter sizes ========
# The sizes of filter files may vary because of the substitution of local
# filenames, logins, etc.
s/^\d+(?= bytes read from )/ssss/;
# ======== OpenSSL error messages ========
# Different releases of the OpenSSL libraries seem to give different error
# numbers, or handle specific bad conditions in different ways, leading to
# different wording in the error messages, so we cannot compare them.
s/(TLS error on connection (?:from|to) .*? \(SSL_\w+\): error:)(.*)/$1 <<detail omitted>>/;
# ======== Maildir things ========
# timestamp output in maildir processing
s/(timestamp=|\(timestamp_only\): )\d+/$1ddddddd/g;
# maildir delivery files appearing in log lines (in cases of error)
s/writing to(?: file)? tmp\/\d+\.[^.]+\.(\S+)/writing to tmp\/MAILDIR.$1/;
s/renamed tmp\/\d+\.[^.]+\.(\S+) as new\/\d+\.[^.]+\.(\S+)/renamed tmp\/MAILDIR.$1 as new\/MAILDIR.$1/;
# Maildir file names in general
s/\b\d+\.H\d+P\d+\b/dddddddddd.HddddddPddddd/;
# Maildirsize data
if (/^\d+S,\d+C\s*$/)
{
print MUNGED "dddS,dC\n";
while (<IN>)
{
last if !/^\d+ \d+\s*$/;
print MUNGED "ddd d\n";
}
last if !defined $_;
}
# ======== Output from the "fd" program about open descriptors ========
# The statuses seem to be different on different operating systems, but
# at least we'll still be checking the number of open fd's.
s/max fd = \d+/max fd = dddd/;
s/status=0 RDONLY/STATUS/g;
s/status=1 WRONLY/STATUS/g;
s/status=2 RDWR/STATUS/g;
# ======== Contents of spool files ========
# A couple of tests dump the contents of the -H file. The length fields
# will be wrong because of different user names, etc.
s/^\d\d\d(?=[PFS*])/ddd/;
# ==========================================================
# Some munging is specific to the specific file types
# ======== stdout ========
if ($is_stdout)
{
# Skip translate_ip_address in -bP output because it ain't always there
next if /translate_ip_address =/;
# In certain filter tests, remove initial filter lines because they just
# clog up by repetition.
if ($rmfiltertest)
{
next if /^(Sender\staken\sfrom|
Return-path\scopied\sfrom|
Sender\s+=|
Recipient\s+=)/x;
if (/^Testing \S+ filter/)
{
$_ = <IN>; # remove blank line
next;
}
}
}
# ======== stderr ========
elsif ($is_stderr)
{
# The very first line of debugging output will vary
s/^Exim version .*/Exim version x.yz ..../;
# Debugging lines for Exim terminations
s/(?<=^>>>>>>>>>>>>>>>> Exim pid=)\d+(?= terminating)/pppp/;
# IP address lookups use gethostbyname() when IPv6 is not supported,
# and gethostbyname2() or getipnodebyname() when it is.
s/\bgethostbyname2?|\bgetipnodebyname/get[host|ipnode]byname[2]/;
# We have to omit the localhost ::1 address so that all is well in
# the IPv4-only case.
print MUNGED "MUNGED: ::1 will be omitted in what follows\n"
if (/looked up these IP addresses/);
next if /name=localhost address=::1/;
# Various other IPv6 lines must be omitted too
next if /using host_fake_gethostbyname for \S+ \(IPv6\)/;
next if /get\[host\|ipnode\]byname\[2\]\(af=inet6\)/;
next if /DNS lookup of \S+ \(AAAA\) using fakens/;
next if / in dns_ipv4_lookup?/;
if (/DNS lookup of \S+ \(AAAA\) gave NO_DATA/)
{
$_= <IN>; # Gets "returning DNS_NODATA"
next;
}
# Skip tls_advertise_hosts and hosts_require_tls checks when the options
# are unset, because tls ain't always there.
next if /in\s(?:tls_advertise_hosts\?|hosts_require_tls\?)
\sno\s\(option\sunset\)/x;
# Skip auxiliary group lists because they will vary.
next if /auxiliary group list:/;
# Skip "extracted from gecos field" because the gecos field varies
next if /extracted from gecos field/;
# Skip "waiting for data on socket" and "read response data: size=" lines
# because some systems pack more stuff into packets than others.
next if /waiting for data on socket/;
next if /read response data: size=/;
# If Exim is compiled with readline support but it can't find the library
# to load, there will be an extra debug line. Omit it.
next if /failed to load readline:/;
# Some DBM libraries seem to make DBM files on opening with O_RDWR without
# O_CREAT; other's don't. In the latter case there is some debugging output
# which is not present in the former. Skip the relevant lines (there are
# two of them).
if (/TESTSUITE\/spool\/db\/\S+ appears not to exist: trying to create/)
{
$_ = <IN>;
next;
}
# Some tests turn on +expand debugging to check on expansions.
# Unfortunately, the Received: expansion varies, depending on whether TLS
# is compiled or not. So we must remove the relevant debugging if it is.
if (/^condition: def:tls_cipher/)
{
while (<IN>) { last if /^condition: def:sender_address/; }
}
elsif (/^expanding: Received: /)
{
while (<IN>) { last if !/^\s/; }
}
# When Exim is checking the size of directories for maildir, it uses
# the check_dir_size() function to scan directories. Of course, the order
# of the files that are obtained using readdir() varies from system to
# system. We therefore buffer up debugging lines from check_dir_size()
# and sort them before outputting them.
if (/^check_dir_size:/ || /^skipping TESTSUITE\/test-mail\//)
{
push @saved, $_;
}
else
{
if (@saved > 0)
{
print MUNGED "MUNGED: the check_dir_size lines have been sorted " .
"to ensure consistency\n";
@saved = sort(@saved);
print MUNGED @saved;
@saved = ();
}
# Skip some lines that Exim puts out at the start of debugging output
# because they will be different in different binaries.
print MUNGED
unless (/^Berkeley DB: / ||
/^Probably (?:Berkeley DB|ndbm|GDBM)/ ||
/^Authenticators:/ ||
/^Lookups:/ ||
/^Support for:/ ||
/^Routers:/ ||
/^Transports:/ ||
/^log selectors =/ ||
/^cwd=/ ||
/^Fixed never_users:/
);
}
next;
}
# ======== All files other than stderr ========
print MUNGED;
}
close(IN);
return $yield;
}
##################################################
# Subroutine to interact with caller #
##################################################
# Arguments: [0] the prompt string
# [1] if there is a U in the prompt and $force_update is true
# Returns: nothing (it sets $_)
sub interact{
print $_[0];
if ($_[1]) { $_ = "u"; print "... update forced\n"; }
else { $_ = <T>; }
}
##################################################
# Subroutine to compare one output file #
##################################################
# When an Exim server is part of the test, its output is in separate files from
# an Exim client. The server data is concatenated with the client data as part
# of the munging operation.
#
# Arguments: [0] the name of the main raw output file
# [1] the name of the server raw output file or undef
# [2] where to put the munged copy
# [3] the name of the saved file
# [4] TRUE if this is a log file whose deliveries must be sorted
#
# Returns: 0 comparison succeeded or differences to be ignored
# 1 comparison failed; files were updated (=> re-compare)
#
# Does not return if the user replies "Q" to a prompt.
sub check_file{
my($rf,$rsf,$mf,$sf,$sortfile) = @_;
# If there is no saved file, the raw files must either not exist, or be
# empty. The test ! -s is TRUE if the file does not exist or is empty.
if (! -e $sf)
{
return 0 if (! -s $rf && ! -s $rsf);
print "\n";
print "** $rf is not empty\n" if (-s $rf);
print "** $rsf is not empty\n" if (defined $rsf && -s $rsf);
for (;;)
{
print "Continue, Show, or Quit? [Q] ";
$_ = <T>;
tests_exit(1) if /^q?$/i;
return 0 if /^c$/i;
last if (/^s$/);
}
foreach $f ($rf, $rsf)
{
if (defined $f && -s $f)
{
print "\n";
print "------------ $f -----------\n"
if (defined $rf && -s $rf && defined $rsf && -s $rsf);
system("$more $f");
}
}
print "\n";
for (;;)
{
interact("Continue, Update & retry, Quit? [Q] ", $force_update);
tests_exit(1) if /^q?$/i;
return 0 if /^c$/i;
last if (/^u$/i);
}
}
# Control reaches here if either (a) there is a saved file ($sf), or (b) there
# was a request to create a saved file. First, create the munged file from any
# data that does exist.
open(MUNGED, ">$mf") || tests_exit(-1, "Failed to open $mf: $!");
my($truncated) = munge($rf) if -e $rf;
if (defined $rsf && -e $rsf)
{
print MUNGED "\n******** SERVER ********\n";
$truncated |= munge($rsf);
}
close(MUNGED);
# If a saved file exists, do the comparison. There are two awkward cases:
#
# If "*** truncated ***" was found in the new file, it means that a log line
# was overlong, and truncated. The problem is that it may be truncated at
# different points on different systems, because of different user name
# lengths. We reload the file and the saved file, and remove lines from the new
# file that precede "*** truncated ***" until we reach one that matches the
# line that precedes it in the saved file.
#
# If $sortfile is set, we are dealing with a mainlog file where the deliveries
# for an individual message might vary in their order from system to system, as
# a result of parallel deliveries. We load the munged file and sort sequences
# of delivery lines.
if (-e $sf)
{
# Deal with truncated text items
if ($truncated)
{
my(@munged, @saved, $i, $j, $k);
open(MUNGED, "$mf") || tests_exit(-1, "Failed to open $mf: $!");
@munged = <MUNGED>;
close(MUNGED);
open(SAVED, "$sf") || tests_exit(-1, "Failed to open $sf: $!");
@saved = <SAVED>;
close(SAVED);
$j = 0;
for ($i = 0; $i < @munged; $i++)
{
if ($munged[$i] =~ /\*\*\* truncated \*\*\*/)
{
for (; $j < @saved; $j++)
{ last if $saved[$j] =~ /\*\*\* truncated \*\*\*/; }
last if $j >= @saved; # not found in saved
for ($k = $i - 1; $k >= 0; $k--)
{ last if $munged[$k] eq $saved[$j - 1]; }
last if $k <= 0; # failed to find previous match
splice @munged, $k + 1, $i - $k - 1;
$i = $k + 1;
}
}
open(MUNGED, ">$mf") || tests_exit(-1, "Failed to open $mf: $!");
for ($i = 0; $i < @munged; $i++)
{ print MUNGED $munged[$i]; }
close(MUNGED);
}
# Deal with log sorting
if ($sortfile)
{
my(@munged, $i, $j);
open(MUNGED, "$mf") || tests_exit(-1, "Failed to open $mf: $!");
@munged = <MUNGED>;
close(MUNGED);
for ($i = 0; $i < @munged; $i++)
{
if ($munged[$i] =~ /^[-\d]{10}\s[:\d]{8}\s[-A-Za-z\d]{16}\s[-=*]>/)
{
for ($j = $i + 1; $j < @munged; $j++)
{
last if $munged[$j] !~
/^[-\d]{10}\s[:\d]{8}\s[-A-Za-z\d]{16}\s[-=*]>/;
}
@temp = splice(@munged, $i, $j - $i);
@temp = sort(@temp);
splice(@munged, $i, 0, @temp);
}
}
open(MUNGED, ">$mf") || tests_exit(-1, "Failed to open $mf: $!");
print MUNGED "**NOTE: The delivery lines in this file have been sorted.\n";
for ($i = 0; $i < @munged; $i++)
{ print MUNGED $munged[$i]; }
close(MUNGED);
}
# Do the comparison
return 0 if (system("$cf $mf $sf >test-cf") == 0);
# Handle comparison failure
print "** Comparison of $mf with $sf failed";
system("$more test-cf");
print "\n";
for (;;)
{
interact("Continue, Update & retry, Quit? [Q] ", $force_update);
tests_exit(1) if /^q?$/i;
return 0 if /^c$/i;
last if (/^u$/i);
}
}
# Update or delete the saved file, and give the appropriate return code.
if (-s $mf)
{ tests_exit(-1, "Failed to cp $mf $sf") if system("cp $mf $sf") != 0; }
else
{ tests_exit(-1, "Failed to unlink $sf") if !unlink($sf); }
return 1;
}
##################################################
# Subroutine to check the output of a test #
##################################################
# This function is called when the series of subtests is complete. It makes
# use of check() file, whose arguments are:
#
# [0] the name of the main raw output file
# [1] the name of the server raw output file or undef
# [2] where to put the munged copy
# [3] the name of the saved file
# [4] TRUE if this is a log file whose deliveries must be sorted
#
# Arguments: none
# Returns: 0 if the output compared equal
# 1 if files were updated and the test must be re-run
sub check_output{
my($yield) = 0;
$yield = 1 if check_file("spool/log/paniclog",
"spool/log/serverpaniclog",
"test-paniclog-munged",
"paniclog/$testno", 0);
$yield = 1 if check_file("spool/log/rejectlog",
"spool/log/serverrejectlog",
"test-rejectlog-munged",
"rejectlog/$testno", 0);
$yield = 1 if check_file("spool/log/mainlog",
"spool/log/servermainlog",
"test-mainlog-munged",
"log/$testno", $sortlog);
if (!$stdout_skip)
{
$yield = 1 if check_file("test-stdout",
"test-stdout-server",
"test-stdout-munged",
"stdout/$testno", 0);
}
if (!$stderr_skip)
{
$yield = 1 if check_file("test-stderr",
"test-stderr-server",
"test-stderr-munged",
"stderr/$testno", 0);
}
# Compare any delivered messages, unless this test is skipped.
if (! $message_skip)
{
my($msgno) = 0;
# Get a list of expected mailbox files for this script. We don't bother with
# directories, just the files within them.
foreach $oldmail (@oldmails)
{
next unless $oldmail =~ /^mail\/$testno\./;
print ">> EXPECT $oldmail\n" if $debug;
$expected_mails{$oldmail} = 1;
}
# If there are any files in test-mail, compare them. Note that "." and
# ".." are automatically omitted by list_files_below().
@mails = list_files_below("test-mail");
foreach $mail (@mails)
{
next if $mail eq "test-mail/oncelog";
$saved_mail = substr($mail, 10); # Remove "test-mail/"
$saved_mail =~ s/^$parm_caller(\/|$)/CALLER/; # Convert caller name
if ($saved_mail =~ /(\d+\.[^.]+\.)/)
{
$msgno++;
$saved_mail =~ s/(\d+\.[^.]+\.)/$msgno./gx;
}
print ">> COMPARE $mail mail/$testno.$saved_mail\n" if $debug;
$yield = 1 if check_file($mail, undef, "test-mail-munged",
"mail/$testno.$saved_mail", 0);
delete $expected_mails{"mail/$testno.$saved_mail"};
}
# Complain if not all expected mails have been found
if (scalar(keys %expected_mails) != 0)
{
foreach $key (keys %expected_mails)
{ print "** no test file found for $key\n"; }
for (;;)
{
interact("Continue, Update & retry, or Quit? [Q] ", $force_update);
tests_exit(1) if /^q?$/i;
last if /^c$/i;
# For update, we not only have to unlink the file, but we must also
# remove it from the @oldmails vector, as otherwise it will still be
# checked for when we re-run the test.
if (/^u$/i)
{
foreach $key (keys %expected_mails)
{
my($i);
tests_exit(-1, "Failed to unlink $key") if !unlink("$key");
for ($i = 0; $i < @oldmails; $i++)
{
if ($oldmails[$i] eq $key)
{
splice @oldmails, $i, 1;
last;
}
}
}
last;
}
}
}
}
# Compare any remaining message logs, unless this test is skipped.
if (! $msglog_skip)
{
# Get a list of expected msglog files for this test
foreach $oldmsglog (@oldmsglogs)
{
next unless $oldmsglog =~ /^$testno\./;
$expected_msglogs{$oldmsglog} = 1;
}
# If there are any files in spool/msglog, compare them. However, we have
# to munge the file names because they are message ids, which are
# time dependent.
if (opendir(DIR, "spool/msglog"))
{
@msglogs = sort readdir(DIR);
closedir(DIR);
foreach $msglog (@msglogs)
{
next if ($msglog eq "." || $msglog eq ".." || $msglog eq "CVS");
($munged_msglog = $msglog) =~
s/((?:[^\W_]{6}-){2}[^\W_]{2})
/new_value($1, "10Hm%s-0005vi-00", \$next_msgid)/egx;
$yield = 1 if check_file("spool/msglog/$msglog", undef,
"test-msglog-munged", "msglog/$testno.$munged_msglog", 0);
delete $expected_msglogs{"$testno.$munged_msglog"};
}
}
# Complain if not all expected msglogs have been found
if (scalar(keys %expected_msglogs) != 0)
{
foreach $key (keys %expected_msglogs)
{
print "** no test msglog found for msglog/$key\n";
($msgid) = $key =~ /^\d+\.(.*)$/;
foreach $cachekey (keys %cache)
{
if ($cache{$cachekey} eq $msgid)
{
print "** original msgid $cachekey\n";
last;
}
}
}
for (;;)
{
interact("Continue, Update, or Quit? [Q] ", $force_update);
tests_exit(1) if /^q?$/i;
last if /^c$/i;
if (/^u$/i)
{
foreach $key (keys %expected_msglogs)
{
tests_exit(-1, "Failed to unlink msglog/$key")
if !unlink("msglog/$key");
}
last;
}
}
}
}
return $yield;
}
##################################################
# Subroutine to run one "system" command #
##################################################
# We put this in a subroutine so that the command can be reflected when
# debugging.
#
# Argument: the command to be run
# Returns: nothing
sub run_system {
my($cmd) = $_[0];
if ($debug)
{
my($prcmd) = $cmd;
$prcmd =~ s/; /;\n>> /;
print ">> $prcmd\n";
}
system("$cmd");
}
##################################################
# Subroutine to run one script command #
##################################################
# The <SCRIPT> file is open for us to read an optional return code line,
# followed by the command line and any following data lines for stdin. The
# command line can be continued by the use of \. Data lines are not continued
# in this way. In all lines, the following substutions are made:
#
# DIR => the current directory
# CALLER => the caller of this script
#
# Arguments: the current test number
# reference to the subtest number, holding previous value
# reference to the expected return code value
# reference to where to put the command name (for messages)
#
# Returns: 0 the commmand was executed inline, no subprocess was run
# 1 a non-exim command was run and waited for
# 2 an exim command was run and waited for
# 3 a command was run and not waited for (daemon, server, exim_lock)
# 4 EOF was encountered after an initial return code line
sub run_command{
my($testno) = $_[0];
my($subtestref) = $_[1];
my($commandnameref) = $_[3];
my($yield) = 1;
if (/^(\d+)\s*$/) # Handle unusual return code
{
my($r) = $_[2];
$$r = $1 << 8;
$_ = <SCRIPT>;
return 4 if !defined $_; # Missing command
$lineno++;
}
chomp;
$wait_time = 0;
# Handle concatenated command lines
s/\s+$//;
while (substr($_, -1) eq"\\")
{
my($temp);
$_ = substr($_, 0, -1);
chomp($temp = <SCRIPT>);
if (defined $temp)
{
$lineno++;
$temp =~ s/\s+$//;
$temp =~ s/^\s+//;
$_ .= $temp;
}
}
# Do substitutions
do_substitute($testno);
if ($debug) { printf ">> $_\n"; }
# Pass back the command name (for messages)
($$commandnameref) = /^(\S+)/;
# Here follows code for handling the various different commands that are
# supported by this script. The first group of commands are all freestanding
# in that they share no common code and are not followed by any data lines.
###################
###################
# The "dbmbuild" command runs exim_dbmbuild. This is used both to test the
# utility and to make DBM files for testing DBM lookups.
if (/^dbmbuild\s+(\S+)\s+(\S+)/)
{
run_system("(./eximdir/exim_dbmbuild $parm_cwd/$1 $parm_cwd/$2;" .
"echo exim_dbmbuild exit code = \$?)" .
">>test-stdout");
return 1;
}
# The "dump" command runs exim_dumpdb. On different systems, the output for
# some types of dump may appear in a different order because it's just hauled
# out of the DBM file. We can solve this by sorting. Ignore the leading
# date/time, as it will be flattened later during munging.
if (/^dump\s+(\S+)/)
{
my($which) = $1;
my(@temp);
print ">> ./eximdir/exim_dumpdb $parm_cwd/spool $which\n" if $debug;
open(IN, "./eximdir/exim_dumpdb $parm_cwd/spool $which |");
@temp = <IN>;
close(IN);
if ($which eq "callout")
{
@temp = sort {
my($aa) = substr $a, 21;
my($bb) = substr $b, 21;
return $aa cmp $bb;
} @temp;
}
open(OUT, ">>test-stdout");
print OUT "+++++++++++++++++++++++++++\n";
print OUT @temp;
close(OUT);
return 1;
}
# The "echo" command is a way of writing comments to the screen.
if (/^echo\s+(.*)$/)
{
print "$1\n";
return 0;
}
# The "exim_lock" command runs exim_lock in the same manner as "server",
# but it doesn't use any input.
if (/^exim_lock\s+(.*)$/)
{
$cmd = "./eximdir/exim_lock $1 >>test-stdout";
$server_pid = open SERVERCMD, "|$cmd" ||
tests_exit(-1, "Failed to run $cmd\n");
# This gives the process time to get started; otherwise the next
# process may not find it there when it expects it.
select(undef, undef, undef, 0.01);
return 3;
}
# The "exinext" command runs exinext
if (/^exinext\s+(.*)/)
{
run_system("(./eximdir/exinext " .
"-DEXIM_PATH=$parm_cwd/eximdir/exim " .
"-C $parm_cwd/test-config $1;" .
"echo exinext exit code = \$?)" .
">>test-stdout");
return 1;
}
# The "gnutls" command makes a copy of saved GnuTLS parameter data in the
# spool directory, to save Exim from re-creating it each time.
if (/^gnutls/)
{
run_system "sudo cp -p aux-fixed/gnutls-params spool/gnutls-params;" .
"sudo chown $parm_eximuser:$parm_eximgroup spool/gnutls-params;" .
"sudo chmod 0400 spool/gnutls-params";
return 1;
}
# The "killdaemon" command should ultimately follow the starting of any Exim
# daemon with the -bd option. We kill with SIGINT rather than SIGTERM to stop
# it outputting "Terminated" to the terminal when not in the background.
if (/^killdaemon/)
{
$pid = `cat $parm_cwd/spool/exim-daemon.*`;
run_system("sudo /bin/kill -SIGINT $pid");
close DAEMONCMD; # Waits for process
run_system("sudo /bin/rm -f spool/exim-daemon.*");
return 1;
}
# The "millisleep" command is like "sleep" except that its argument is in
# milliseconds, thus allowing for a subsecond sleep, which is, in fact, all it
# is used for.
elsif (/^millisleep\s+(.*)$/)
{
select(undef, undef, undef, $1/1000);
return 0;
}
# The "sleep" command does just that. For sleeps longer than 1 second we
# tell the user what's going on.
if (/^sleep\s+(.*)$/)
{
if ($1 == 1)
{
sleep(1);
}
else
{
printf(" Test %d sleep $1 ", $$subtestref);
for (1..$1)
{
print ".";
sleep(1);
}
printf("\r Test %d $cr", $$subtestref);
}
return 0;
}
# Various Unix management commands are recognized
if (/^(ln|ls|du|mkdir|mkfifo|touch|cp)\s/ ||
/^sudo (rmdir|rm|chown|chmod)\s/)
{
run_system("$_ >>test-stdout 2>>test-stderr");
return 1;
}
###################
###################
# The next group of commands are also freestanding, but they are all followed
# by data lines.
# The "server" command starts up a script-driven server that runs in parallel
# with the following exim command. Therefore, we want to run a subprocess and
# not yet wait for it to complete. The waiting happens after the next exim
# command, triggered by $server_pid being non-zero. The server sends its output
# to a different file. The variable $server_opts, if not empty, contains
# options to disable IPv4 or IPv6 if necessary.
if (/^server\s+(.*)$/)
{
$cmd = "./bin/server $server_opts $1 >>test-stdout-server";
print ">> $cmd\n" if ($debug);
$server_pid = open SERVERCMD, "|$cmd" || tests_exit(-1, "Failed to run $cmd");
SERVERCMD->autoflush(1);
print ">> Server pid is $server_pid\n" if $debug;
while (<SCRIPT>)
{
$lineno++;
last if /^\*{4}\s*$/;
print SERVERCMD;
}
print SERVERCMD "++++\n"; # Send end to server; can't send EOF yet
# because close() waits for the process.
# This gives the server time to get started; otherwise the next
# process may not find it there when it expects it.
select(undef, undef, undef, 0.01);
return 3;
}
# The "write" command is a way of creating files of specific sizes for
# buffering tests, or containing specific data lines from within the script
# (rather than hold lots of little files). The "catwrite" command does the
# same, but it also copies the lines to test-stdout.
if (/^(cat)?write\s+(\S+)(?:\s+(.*))?\s*$/)
{
my($cat) = defined $1;
@sizes = ();
@sizes = split /\s+/, $3 if defined $3;
open FILE, ">$2" || tests_exit(-1, "Failed to open \"$2\": $!");
if ($cat)
{
open CAT, ">>test-stdout" ||
tests_exit(-1, "Failed to open test-stdout: $!");
print CAT "==========\n";
}
if (scalar @sizes > 0)
{
# Pre-data
while (<SCRIPT>)
{
$lineno++;
last if /^\+{4}\s*$/;
print FILE;
print CAT if $cat;
}
# Sized data
while (scalar @sizes > 0)
{
($count,$len,$leadin) = (shift @sizes) =~ /(\d+)x(\d+)(?:=(.*))?/;
$leadin = "" if !defined $leadin;
$leadin =~ s/_/ /g;
$len -= length($leadin) + 1;
while ($count-- > 0)
{
print FILE $leadin, "a" x $len, "\n";
print CAT $leadin, "a" x $len, "\n" if $cat;
}
}
}
# Post data, or only data if no sized data
while (<SCRIPT>)
{
$lineno++;
last if /^\*{4}\s*$/;
print FILE;
print CAT if $cat;
}
close FILE;
if ($cat)
{
print CAT "==========\n";
close CAT;
}
return 0;
}
###################
###################
# From this point on, script commands are implemented by setting up a shell
# command in the variable $cmd. Shared code to run this command and handle its
# input and output follows.
# The "client" and "client-ssl" commands run a script-driven program that plays
# the part of an email client. We also have the availability of running Perl
# for doing one-off special things.
if (/^client/ || /^client-ssl/ || /^(sudo\s+)?perl\b/)
{
s"client"./bin/client";
$cmd = "$_ >>test-stdout 2>>test-stderr";
}
# For the "exim" command, replace the text "exim" with the path for the test
# binary, plus -D options to pass over various parameters, and a -C option for
# the testing configuration file. When running in the test harness, Exim does
# not drop privilege when -C and -D options are present. To run the exim
# command as root, we use sudo.
elsif (/^([A-Z_]+=\S+\s+)?(\d+)?\s*(sudo\s+)?exim(_\S+)?\s+(.*)$/)
{
$args = $5;
my($envset) = (defined $1)? $1 : "";
my($sudo) = (defined $3)? "sudo " : "";
my($special)= (defined $4)? $4 : "";
$wait_time = (defined $2)? $2 : 0;
# Return 2 rather than 1 afterwards
$yield = 2;
# Update the test number
$$subtestref = $$subtestref + 1;
printf(" Test %d $cr", $$subtestref);
# Copy the configuration file, making the usual substitutions.
open (IN, "$parm_cwd/confs/$testno") ||
tests_exit(-1, "Couldn't open $parm_cwd/confs/$testno: $!\n");
open (OUT, ">test-config") ||
tests_exit(-1, "Couldn't open test-config: $!\n");
while (<IN>)
{
do_substitute($testno);
print OUT;
}
close(IN);
close(OUT);
# The string $msg1 in args substitutes the message id of the first
# message on the queue, and so on. */
if ($args =~ /\$msg/)
{
my($listcmd) = "$parm_cwd/eximdir/exim -bp " .
"-DEXIM_PATH=$parm_cwd/eximdir/exim " .
"-C $parm_cwd/test-config |";
print ">> Getting queue list from:\n>> $listcmd\n" if ($debug);
open (QLIST, $listcmd) || tests_exit(-1, "Couldn't run \"exim -bp\": $!\n");
my(@msglist) = ();
while (<QLIST>) { push (@msglist, $1) if /^\s*\d+[smhdw]\s+\S+\s+(\S+)/; }
close(QLIST);
# Done backwards just in case there are more than 9
my($i);
for ($i = @msglist; $i > 0; $i--) { $args =~ s/\$msg$i/$msglist[$i-1]/g; }
}
# If -d is specified in $optargs, remove it from $args; i.e. let
# the command line for runtest override. Then run Exim.
$args =~ s/(?:^|\s)-d\S*// if $optargs =~ /(?:^|\s)-d/;
$cmd = "$envset$sudo$parm_cwd/eximdir/exim$special$optargs " .
"-DEXIM_PATH=$parm_cwd/eximdir/exim$special " .
"-C $parm_cwd/test-config $args " .
">>test-stdout 2>>test-stderr";
# If the command is starting an Exim daemon, we run it in the same
# way as the "server" command above, that is, we don't want to wait
# for the process to finish. That happens when "killdaemon" is obeyed later
# in the script. We also send the stderr output to test-stderr-server. The
# daemon has its log files put in a different place too (by configuring with
# log_file_path). This requires the directory to be set up in advance.
#
# There are also times when we want to run a non-daemon version of Exim
# (e.g. a queue runner) with the server configuration. In this case,
# we also define -DNOTDAEMON.
if ($cmd =~ /\s-DSERVER=server\s/ && $cmd !~ /\s-DNOTDAEMON\s/)
{
if ($debug) { printf ">> daemon: $cmd\n"; }
run_system("sudo mkdir spool/log 2>/dev/null");
run_system("sudo chown $parm_eximuser:$parm_eximgroup spool/log");
# Before running the command, convert the -bd option into -bdf so that an
# Exim daemon doesn't double fork. This means that when we wait close
# DAEMONCMD, it waits for the correct process.
$cmd =~ s/\s-bd\s/ -bdf /;
print ">> |${cmd}-server\n" if ($debug);
open DAEMONCMD, "|${cmd}-server" || tests_exit(-1, "Failed to run $cmd");
DAEMONCMD->autoflush(1);
while (<SCRIPT>) { $lineno++; last if /^\*{4}\s*$/; } # Ignore any input
select(undef, undef, undef, 0.3); # Let the daemon get going
return 3; # Don't wait
}
}
# Unknown command
else { tests_exit(-1, "Command unrecognized in line $lineno: $_"); }
# Run the command, with stdin connected to a pipe, and write the stdin data
# to it, with appropriate substitutions. If a line ends with \NONL\, chop off
# the terminating newline (and the \NONL\). If the command contains
# -DSERVER=server add "-server" to the command, where it will adjoin the name
# for the stderr file. See comment above about the use of -DSERVER.
$stderrsuffix = ($cmd =~ /\s-DSERVER=server\s/)? "-server" : "";
print ">> |${cmd}${stderrsuffix}\n" if ($debug);
open CMD, "|${cmd}${stderrsuffix}" || tests_exit(1, "Failed to run $cmd");
CMD->autoflush(1);
while (<SCRIPT>)
{
$lineno++;
last if /^\*{4}\s*$/;
do_substitute($testno);
if (/^(.*)\\NONL\\\s*$/) { print CMD $1; } else { print CMD; }
}
# For timeout tests, wait before closing the pipe; we expect a
# SIGPIPE error in this case.
if ($wait_time > 0)
{
printf(" Test %d sleep $wait_time ", $$subtestref);
while ($wait_time-- > 0)
{
print ".";
sleep(1);
}
printf("\r Test %d $cr", $$subtestref);
}
$sigpipehappened = 0;
close CMD; # Waits for command to finish
return $yield; # Ran command and waited
}
###############################################################################
###############################################################################
# Here beginneth the Main Program ...
###############################################################################
###############################################################################
autoflush STDOUT 1;
print "Exim tester $testversion\n";
##################################################
# Check for the "less" command #
##################################################
$more = "more" if system("which less >/dev/null 2>&1") != 0;
##################################################
# Check for sudo access to root #
##################################################
print "You need to have sudo access to root to run these tests. Checking ...\n";
if (system("sudo date >/dev/null") != 0)
{
die "** Test for sudo failed: testing abandoned.\n";
}
else
{
print "Test for sudo OK\n";
}
##################################################
# See if an Exim binary has been given #
##################################################
# If the first character of the first argument is '/', the argument is taken
# as the path to the binary.
$parm_exim = (@ARGV > 0 && $ARGV[0] =~ ?^/?)? shift @ARGV : "";
print "Exim binary is $parm_exim\n" if $parm_exim ne "";
##################################################
# Sort out options and which tests are to be run #
##################################################
# There are a few possible options for the test script itself; after these, any
# options are passed on to Exim calls within the tests. Typically, this is used
# to turn on Exim debugging while setting up a test.
while (@ARGV > 0 && $ARGV[0] =~ /^-/)
{
my($arg) = shift @ARGV;
if ($optargs eq "")
{
if ($arg eq "-DEBUG") { $debug = 1; $cr = "\n"; next; }
if ($arg eq "-DIFF") { $cf = "diff -u"; next; }
if ($arg eq "-UPDATE") { $force_update = 1; next; }
if ($arg eq "-NOIPV4") { $have_ipv4 = 0; next; }
if ($arg eq "-NOIPV6") { $have_ipv6 = 0; next; }
if ($arg eq "-KEEP") { $save_output = 1; next; }
}
$optargs .= " $arg";
}
# Any subsequent arguments are a range of test numbers.
if (@ARGV > 0)
{
$test_end = $test_start = $ARGV[0];
$test_end = $ARGV[1] if (@ARGV > 1);
$test_end = ($test_start >= 9000)? $test_special_top : $test_top
if $test_end eq "+";
die "** Test numbers out of order\n" if ($test_end < $test_start);
}
##################################################
# Make the command's directory current #
##################################################
# After doing so, we find its absolute path name.
$cwd = $0;
$cwd = '.' if ($cwd !~ s|/[^/]+$||);
chdir($cwd) || die "** Failed to chdir to \"$cwd\": $!\n";
$parm_cwd = Cwd::getcwd();
##################################################
# Search for an Exim binary to test #
##################################################
# If an Exim binary hasn't been provided, try to find one. We can handle the
# case where exim-testsuite is installed alongside Exim source directories. For
# PH's private convenience, if there's a directory just called "exim4", that
# takes precedence; otherwise exim-snapshot takes precedence over any numbered
# releases.
if ($parm_exim eq "")
{
my($use_srcdir) = "";
opendir DIR, ".." || die "** Failed to opendir \"..\": $!\n";
while ($f = readdir(DIR))
{
my($srcdir);
# Try this directory if it is "exim4" or if it is exim-snapshot or exim-n.m
# possibly followed by -RCx where n.m is greater than any previously tried
# directory. Thus, we should choose the highest version of Exim that has
# been compiled.
if ($f eq "exim4" || $f eq "exim-snapshot")
{ $srcdir = $f; }
else
{ $srcdir = $f
if ($f =~ /^exim-\d+\.\d+(-RC\d+)?$/ && $f gt $use_srcdir); }
# Look for a build directory with a binary in it. If we find a binary,
# accept this source directory.
if ($srcdir)
{
opendir SRCDIR, "../$srcdir" ||
die "** Failed to opendir \"$cwd/../$srcdir\": $!\n";
while ($f = readdir(SRCDIR))
{
if ($f =~ /^build-/ && -e "../$srcdir/$f/exim")
{
$use_srcdir = $srcdir;
$parm_exim = "$cwd/../$srcdir/$f/exim";
$parm_exim =~ s'/[^/]+/\.\./'/';
last;
}
}
closedir(SRCDIR);
}
# If we have found "exim4" or "exim-snapshot", that takes precedence.
# Otherwise, continue to see if there's a later version.
last if $use_srcdir eq "exim4" || $use_srcdir eq "exim-snapshot";
}
closedir(DIR);
print "Exim binary found in $parm_exim\n" if $parm_exim ne "";
}
# If $parm_exim is still empty, ask the caller
if ($parm_exim eq "")
{
print "** Did not find an Exim binary to test\n";
for ($i = 0; $i < 5; $i++)
{
my($trybin);
print "** Enter pathname for Exim binary: ";
chomp($trybin = <STDIN>);
if (-e $trybin)
{
$parm_exim = $trybin;
last;
}
else
{
print "** $trybin does not exist\n";
}
}
die "** Too many tries\n" if $parm_exim eq "";
}
##################################################
# Find what is in the binary #
##################################################
open(EXIMINFO, "$parm_exim -C confs/0000 -DDIR=$parm_cwd " .
"-bP exim_user exim_group|") ||
die "** Cannot run $parm_exim: $!\n";
while(<EXIMINFO>)
{
$parm_eximuser = $1 if /^exim_user = (.*)$/;
$parm_eximgroup = $1 if /^exim_group = (.*)$/;
}
close(EXIMINFO);
if (defined $parm_eximuser)
{
if ($parm_eximuser =~ /^\d+$/) { $parm_exim_uid = $parm_eximuser; }
else { $parm_exim_uid = getpwnam($parm_eximuser); }
}
if (defined $parm_eximgroup)
{
if ($parm_eximgroup =~ /^\d+$/) { $parm_exim_gid = $parm_eximgroup; }
else { $parm_exim_gid = getgrnam($parm_eximgroup); }
}
open(EXIMINFO, "$parm_exim -bV -C confs/0000 -DDIR=$parm_cwd |") ||
die "** Cannot run $parm_exim: $!\n";
print "-" x 78, "\n";
while (<EXIMINFO>)
{
my(@temp);
if (/^Exim version/) { print; next; }
if (/^Support for: (.*)/)
{
print;
@temp = split /(\s+)/, $1;
push(@temp, ' ');
%parm_support = @temp;
}
if (/^Lookups: (.*)/)
{
print;
@temp = split /(\s+)/, $1;
push(@temp, ' ');
%parm_lookups = @temp;
}
if (/^Authenticators: (.*)/)
{
print;
@temp = split /(\s+)/, $1;
push(@temp, ' ');
%parm_authenticators = @temp;
}
if (/^Routers: (.*)/)
{
print;
@temp = split /(\s+)/, $1;
push(@temp, ' ');
%parm_routers = @temp;
}
# Some transports have options, e.g. appendfile/maildir. For those, ensure
# that the basic transport name is set, and then the name with each of the
# options.
if (/^Transports: (.*)/)
{
print;
@temp = split /(\s+)/, $1;
my($i,$k);
push(@temp, ' ');
%parm_transports = @temp;
foreach $k (keys %parm_transports)
{
if ($k =~ "/")
{
@temp = split /\//, $k;
$parm_transports{"$temp[0]"} = " ";
for ($i = 1; $i < @temp; $i++)
{ $parm_transports{"$temp[0]/$temp[$i]"} = " "; }
}
}
}
}
close(EXIMINFO);
print "-" x 78, "\n";
##################################################
# Check for SpamAssassin and ClamAV #
##################################################
# These are crude tests. If they aren't good enough, we'll have to improve
# them, for example by actually passing a message through spamc or clamscan.
if (defined $parm_support{'Content_Scanning'})
{
if (system("spamc -h 2>/dev/null >/dev/null") == 0)
{
$parm_running{'SpamAssassin'} = ' ';
print "The spamc command works:\n";
# This test for an active SpamAssassin is courtesy of John Jetmore.
# The tests are hard coded to localhost:783, so no point in making
# this test flexible like the clamav test until the test scripts are
# changed. spamd doesn't have the nice PING/PONG protoccol that
# clamd does, but it does respond to errors in an informative manner,
# so use that.
my($sint,$sport) = ('127.0.0.1',783);
eval
{
my $sin = sockaddr_in($sport, inet_aton($sint))
or die "** Failed packing $sint:$sport\n";
socket(SOCK, PF_INET, SOCK_STREAM, getprotobyname('tcp'))
or die "** Unable to open socket $sint:$sport\n";
local $SIG{ALRM} =
sub { die "** Timeout while connecting to socket $sint:$sport\n"; };
alarm(5);
connect(SOCK, $sin)
or die "** Unable to connect to socket $sint:$sport\n";
alarm(0);
select((select(SOCK), $| = 1)[0]);
print SOCK "bad command\r\n";
$SIG{ALRM} =
sub { die "** Timeout while reading from socket $sint:$sport\n"; };
alarm(10);
my $res = <SOCK>;
alarm(0);
$res =~ m|^SPAMD/|
or die "** Did not get SPAMD from socket $sint:$sport. "
."It said: $res\n";
};
alarm(0);
if($@)
{
print " $@";
print " Assume SpamAssassin (spamd) is not running\n";
}
else
{
$parm_running{'SpamAssassin'} = ' ';
print " SpamAssassin (spamd) seems to be running\n";
}
}
else
{
print "The spamc command failed: assume SpamAssassin (spamd) is not running\n";
}
# For ClamAV, we need to find the clamd socket for use in the Exim
# configuration. Search for the clamd configuration file.
if (system("clamscan -h 2>/dev/null >/dev/null") == 0)
{
my($f, $clamconf, $test_prefix);
print "The clamscan command works";
$test_prefix = $ENV{EXIM_TEST_PREFIX};
$test_prefix = "" if !defined $test_prefix;
foreach $f ("$test_prefix/etc/clamd.conf",
"$test_prefix/usr/local/etc/clamd.conf",
"$test_prefix/etc/clamav/clamd.conf", "")
{
if (-e $f)
{
$clamconf = $f;
last;
}
}
if ($clamconf ne "")
{
open(IN, "$clamconf") || die "\n** Unable to open $clamconf: $!\n";
while (<IN>)
{
if (/^LocalSocket\s+(.*)/)
{
$parm_clamsocket = $1;
last;
}
}
close(IN);
if (-e $parm_clamsocket)
{
print ":\n The clamd socket is $parm_clamsocket\n";
# This test for an active ClamAV is courtesy of Daniel Tiefnig.
eval
{
my $sun = sockaddr_un($parm_clamsocket) or die "** Failed packing '$parm_clamsocket'\n";
socket(SOCK, AF_UNIX, SOCK_STREAM, 0) or die "** Unable to open socket '$parm_clamsocket'\n";
local $SIG{ALRM} = sub { die "** Timeout while connecting to socket '$parm_clamsocket'\n"; };
alarm(5);
connect(SOCK, $sun) or die "** Unable to connect to socket '$parm_clamsocket'\n";
alarm(0);
my $ofh = select SOCK; $| = 1; select $ofh;
print SOCK "PING\n";
$SIG{ALRM} = sub { die "** Timeout while reading from socket '$parm_clamsocket'\n"; };
alarm(10);
my $res = <SOCK>;
alarm(0);
$res =~ /PONG/ or die "** Did not get PONG from socket '$parm_clamsocket'. It said: $res\n";
};
alarm(0);
if($@)
{
warn $@;
print " Assume ClamAV is not running\n";
}
else
{
$parm_running{'ClamAV'} = ' ';
print " ClamAV seems to be running\n";
}
}
else
{
print ", but the socket for clamd does not exist\n";
print "Assume ClamAV is not running\n";
}
}
else
{
print ", but I can't find a configuration for clamd\n";
print "Assume ClamAV is not running\n";
}
}
}
##################################################
# Test for the basic requirements #
##################################################
# This test suite assumes that Exim has been built with at least the "usual"
# set of routers, transports, and lookups. Ensure that this is so.
$missing = "";
$missing .= " Lookup: lsearch\n" if (!defined $parm_lookups{'lsearch'});
$missing .= " Router: accept\n" if (!defined $parm_routers{'accept'});
$missing .= " Router: dnslookup\n" if (!defined $parm_routers{'dnslookup'});
$missing .= " Router: manualroute\n" if (!defined $parm_routers{'manualroute'});
$missing .= " Router: redirect\n" if (!defined $parm_routers{'redirect'});
$missing .= " Transport: appendfile\n" if (!defined $parm_transports{'appendfile'});
$missing .= " Transport: autoreply\n" if (!defined $parm_transports{'autoreply'});
$missing .= " Transport: pipe\n" if (!defined $parm_transports{'pipe'});
$missing .= " Transport: smtp\n" if (!defined $parm_transports{'smtp'});
if ($missing ne "")
{
print "\n";
print "** Many features can be included or excluded from Exim binaries.\n";
print "** This test suite requires that Exim is built to contain a certain\n";
print "** set of basic facilities. It seems that some of these are missing\n";
print "** from the binary that is under test, so the test cannot proceed.\n";
print "** The missing facilities are:\n";
print "$missing";
die "** Test script abandoned\n";
}
##################################################
# Check for the auxiliary programs #
##################################################
# These are always required:
for $prog ("cf", "checkaccess", "client", "client-ssl", "client-gnutls",
"fakens", "iefbr14", "server")
{
next if ($prog eq "client-ssl" && !defined $parm_support{'OpenSSL'});
next if ($prog eq "client-gnutls" && !defined $parm_support{'GnuTLS'});
if (!-e "bin/$prog")
{
print "\n";
print "** bin/$prog does not exist. Have you run ./configure and make?\n";
die "** Test script abandoned\n";
}
}
# If the "loaded" binary is missing, we cut out tests for ${dlfunc. It isn't
# compiled on systems where we don't know how to. However, if Exim does not
# have that functionality compiled, we needn't bother.
$dlfunc_deleted = 0;
if (defined $parm_support{'Expand_dlfunc'} && !-e "bin/loaded")
{
delete $parm_support{'Expand_dlfunc'};
$dlfunc_deleted = 1;
}
##################################################
# Find environmental details #
##################################################
# Find the caller of this program.
($parm_caller,$pwpw,$parm_caller_uid,$parm_caller_gid,$pwquota,$pwcomm,
$pwgecos, $parm_caller_home) = getpwuid($>);
$pwpw = $pwpw; # Kill Perl warnings
$pwquota = $pwquota;
$pwcomm = $pwcomm;
$pwgecos = $pwgecos;
$parm_caller_group = getgrgid($parm_caller_gid);
print "Program caller is $parm_caller, whose group is $parm_caller_group\n";
print "Home directory is $parm_caller_home\n";
print "You need to be in the Exim group to run these tests. Checking ...";
if (`groups` =~ /\b\Q$parm_eximgroup\E\b/)
{
print " OK\n";
}
else
{
print "\nOh dear, you are not in the Exim group.\n";
die "** Testing abandoned.\n";
}
# Find this host's IP addresses - there may be many, of course, but we keep
# one of each type (IPv4 and IPv6).
$parm_ipv4 = "";
$parm_ipv6 = "";
$local_ipv4 = "";
$local_ipv6 = "";
open(IFCONFIG, "ifconfig -a|") || die "** Cannot run \"ifconfig\": $!\n";
while (($parm_ipv4 eq "" || $parm_ipv6 eq "") && ($_ = <IFCONFIG>))
{
my($ip);
if ($parm_ipv4 eq "" &&
$_ =~ /^\s*inet(?:\saddr)?:?\s?(\d+\.\d+\.\d+\.\d+)\s/i)
{
$ip = $1;
next if ($ip eq "127.0.0.1");
$parm_ipv4 = $ip;
}
if ($parm_ipv6 eq "" &&
$_ =~ /^\s*inet6(?:\saddr)?:?\s?([abcdef\d:]+)/i)
{
$ip = $1;
next if ($ip eq "::1" || $ip =~ /^fe80/i);
$parm_ipv6 = $ip;
}
}
close(IFCONFIG);
# Use private IP addresses if there are no public ones.
$parm_ipv4 = $local_ipv4 if ($parm_ipv4 eq "");
$parm_ipv6 = $local_ipv6 if ($parm_ipv6 eq "");
# If either type of IP address is missing, we need to set the value to
# something other than empty, because that wrecks the substitutions. The value
# is reflected, so use a meaningful string. Set appropriate options for the
# "server" command. In practice, however, many tests assume 127.0.0.1 is
# available, so things will go wrong if there is no IPv4 address. The lack
# of IPV4 or IPv6 can be simulated by command options, which force $have_ipv4
# and $have_ipv6 false.
if ($parm_ipv4 eq "")
{
$have_ipv4 = 0;
$parm_ipv4 = "<no IPv4 address found>";
$server_opts .= " -noipv4";
}
elsif ($have_ipv4 == 0)
{
$parm_ipv4 = "<IPv4 testing disabled>";
$server_opts .= " -noipv4";
}
else
{
$parm_running{"IPv4"} = " ";
}
if ($parm_ipv6 eq "")
{
$have_ipv6 = 0;
$parm_ipv6 = "<no IPv6 address found>";
$server_opts .= " -noipv6";
delete($parm_support{"IPv6"});
}
elsif ($have_ipv6 == 0)
{
$parm_ipv6 = "<IPv6 testing disabled>";
$server_opts .= " -noipv6";
delete($parm_support{"IPv6"});
}
elsif (!defined $parm_support{'IPv6'})
{
$have_ipv6 = 0;
$parm_ipv6 = "<no IPv6 support in Exim binary>";
$server_opts .= " -noipv6";
}
else
{
$parm_running{"IPv6"} = " ";
}
print "IPv4 address is $parm_ipv4\n";
print "IPv6 address is $parm_ipv6\n";
# Find the host name, fully qualified.
chomp($temp = `hostname`);
$parm_hostname = (gethostbyname($temp))[0];
$parm_hostname = "no.host.name.found" if $parm_hostname eq "";
print "Hostname is $parm_hostname\n";
if ($parm_hostname !~ /\./)
{
print "\n*** Host name is not fully qualified: this may cause problems ***\n\n";
}
# Find the user's shell
$parm_shell = $ENV{'SHELL'};
##################################################
# Create a testing version of Exim #
##################################################
# We want to be able to run Exim with a variety of configurations. Normally,
# the use of -C to change configuration causes Exim to give up its root
# privilege (unless the caller is exim or root). For these tests, we do not
# want this to happen. Also, we want Exim to know that it is running in its
# test harness.
# We achieve this by copying the binary and patching it as we go. The new
# binary knows it is a testing copy, and it allows -C and -D without loss of
# privilege. Clearly, this file is dangerous to have lying around on systems
# where there are general users with login accounts. To protect against this,
# we put the new binary in a special directory that is accessible only to the
# caller of this script, who is known to have sudo root privilege from the test
# that was done above. Furthermore, we ensure that the binary is deleted at the
# end of the test. First ensure the directory exists.
if (-d "eximdir")
{ unlink "eximdir/exim"; } # Just in case
else
{
mkdir("eximdir", 0710) || die "** Unable to mkdir $parm_cwd/eximdir: $!\n";
system("sudo chgrp $parm_eximgroup eximdir");
}
# The construction of the patched binary must be done as root, so we use
# a separate script. As well as indicating that this is a test-harness binary,
# the version number is patched to "x.yz" so that its length is always the
# same. Otherwise, when it appears in Received: headers, it affects the length
# of the message, which breaks certain comparisons.
die "** Unable to make patched exim: $!\n"
if (system("sudo ./patchexim $parm_exim") != 0);
# From this point on, exits from the program must go via the subroutine
# tests_exit(), so that suitable cleaning up can be done when required.
# Arrange to catch interrupting signals, to assist with this.
$SIG{'INT'} = \&inthandler;
$SIG{'PIPE'} = \&pipehandler;
# For some tests, we need another copy of the binary that is setuid exim rather
# than root.
system("sudo cp eximdir/exim eximdir/exim_exim;" .
"sudo chown $parm_eximuser eximdir/exim_exim;" .
"sudo chgrp $parm_eximgroup eximdir/exim_exim;" .
"sudo chmod 06755 eximdir/exim_exim");
##################################################
# Make copies of utilities we might need #
##################################################
# Certain of the tests make use of some of Exim's utilities. We do not need
# to be root to copy these.
($parm_exim_dir) = $parm_exim =~ ?^(.*)/exim?;
$dbm_build_deleted = 0;
if (defined $parm_lookups{'dbm'} &&
system("cp $parm_exim_dir/exim_dbmbuild eximdir") != 0)
{
delete $parm_lookups{'dbm'};
$dbm_build_deleted = 1;
}
if (system("cp $parm_exim_dir/exim_dumpdb eximdir") != 0)
{
tests_exit(-1, "Failed to make a copy of exim_dumpdb: $!");
}
if (system("cp $parm_exim_dir/exim_lock eximdir") != 0)
{
tests_exit(-1, "Failed to make a copy of exim_lock: $!");
}
if (system("cp $parm_exim_dir/exinext eximdir") != 0)
{
tests_exit(-1, "Failed to make a copy of exinext: $!");
}
##################################################
# Check that the Exim user can access stuff #
##################################################
# We delay this test till here so that we can check access to the actual test
# binary. This will be needed when Exim re-exec's itself to do deliveries.
print "Exim user is $parm_eximuser ($parm_exim_uid)\n";
print "Exim group is $parm_eximgroup ($parm_exim_gid)\n";
print "The Exim user needs access to the test suite directory. Checking ...";
if (($rc = system("sudo bin/checkaccess $parm_cwd/eximdir/exim $parm_eximuser $parm_eximgroup")) != 0)
{
my($why) = "unknown failure $rc";
$rc >>= 8;
$why = "Couldn't find user \"$parm_eximuser\"" if $rc == 1;
$why = "Couldn't find group \"$parm_eximgroup\"" if $rc == 2;
$why = "Couldn't read auxiliary group list" if $rc == 3;
$why = "Couldn't get rid of auxiliary groups" if $rc == 4;
$why = "Couldn't set gid" if $rc == 5;
$why = "Couldn't set uid" if $rc == 6;
$why = "Couldn't open \"$parm_cwd/eximdir/exim\"" if $rc == 7;
print "\n** $why\n";
tests_exit(-1, "$parm_eximuser cannot access the test suite directory");
}
else
{
print " OK\n";
}
##################################################
# Create a list of available tests #
##################################################
# The scripts directory contains a number of subdirectories whose names are
# of the form 0000-xxxx, 1100-xxxx, 2000-xxxx, etc. Each set of tests apart
# from the first requires certain optional features to be included in the Exim
# binary. These requirements are contained in a file called "REQUIRES" within
# the directory. We scan all these tests, discarding those that cannot be run
# because the current binary does not support the right facilities, and also
# those that are outside the numerical range selected.
print "\nTest range is $test_start to $test_end\n";
print "Omitting \${dlfunc expansion tests (loadable module not present)\n"
if $dlfunc_deleted;
print "Omitting dbm tests (unable to copy exim_dbmbuild)\n"
if $dbm_build_deleted;
opendir(DIR, "scripts") || tests_exit(-1, "Failed to opendir(\"scripts\"): $!");
@test_dirs = sort readdir(DIR);
closedir(DIR);
for ($i = 0; $i < @test_dirs; $i++)
{
my($testdir) = $test_dirs[$i];
my($wantthis) = 1;
next if $testdir eq "." || $testdir eq "..";
print ">>Checking $testdir\n" if $debug;
# Skip this directory if the first test is equal or greater than the first
# test in the next directory.
next if ($i < @test_dirs - 1) &&
($test_start >= substr($test_dirs[$i+1], 0, 4));
# No need to carry on if the end test is less than the first test in this
# subdirectory.
last if $test_end < substr($testdir, 0, 4);
# Check requirements, if any.
if (open(REQUIRES, "scripts/$testdir/REQUIRES"))
{
while (<REQUIRES>)
{
next if /^\s*$/;
s/\s+$//;
if (/^support (.*)$/)
{
if (!defined $parm_support{$1}) { $wantthis = 0; last; }
}
elsif (/^running (.*)$/)
{
if (!defined $parm_running{$1}) { $wantthis = 0; last; }
}
elsif (/^lookup (.*)$/)
{
if (!defined $parm_lookups{$1}) { $wantthis = 0; last; }
}
elsif (/^authenticators? (.*)$/)
{
if (!defined $parm_authenticators{$1}) { $wantthis = 0; last; }
}
elsif (/^router (.*)$/)
{
if (!defined $parm_routers{$1}) { $wantthis = 0; last; }
}
elsif (/^transport (.*)$/)
{
if (!defined $parm_transports{$1}) { $wantthis = 0; last; }
}
else
{
tests_exit(-1, "Unknown line in \"scripts/$testdir/REQUIRES\": \"$_\"");
}
}
close(REQUIRES);
}
else
{
tests_exit(-1, "Failed to open \"scripts/$testdir/REQUIRES\": $!")
unless $!{ENOENT};
}
# Loop if we do not want the tests in this subdirectory.
if (!$wantthis)
{
chomp;
print "Omitting tests in $testdir (missing $_)\n";
next;
}
# We want the tests from this subdirectory, provided they are in the
# range that was selected.
opendir(SUBDIR, "scripts/$testdir") ||
tests_exit(-1, "Failed to opendir(\"scripts/$testdir\"): $!");
@testlist = sort readdir(SUBDIR);
close(SUBDIR);
foreach $test (@testlist)
{
next if $test !~ /^\d{4}$/;
next if $test < $test_start || $test > $test_end;
push @test_list, "$testdir/$test";
}
}
print ">>Test List: @test_list\n", if $debug;
##################################################
# Munge variable auxiliary data #
##################################################
# Some of the auxiliary data files have to refer to the current testing
# directory and other parameter data. The generic versions of these files are
# stored in the aux-var-src directory. At this point, we copy each of them
# to the aux-var directory, making appropriate substitutions. There aren't very
# many of them, so it's easiest just to do this every time. Ensure the mode
# is standardized, as this path is used as a test for the ${stat: expansion.
# A similar job has to be done for the files in the dnszones-src directory, to
# make the fake DNS zones for testing. Most of the zone files are copied to
# files of the same name, but db.ipv4.V4NET and db.ipv6.V6NET use the testing
# networks that are defined by parameter.
foreach $basedir ("aux-var", "dnszones")
{
system("sudo rm -rf $parm_cwd/$basedir");
mkdir("$parm_cwd/$basedir", 0777);
chmod(0755, "$parm_cwd/$basedir");
opendir(AUX, "$parm_cwd/$basedir-src") ||
tests_exit(-1, "Failed to opendir $parm_cwd/$basedir-src: $!");
my(@filelist) = readdir(AUX);
close(AUX);
foreach $file (@filelist)
{
my($outfile) = $file;
next if $file =~ /^\./;
if ($file eq "db.ip4.V4NET")
{
$outfile = "db.ip4.$parm_ipv4_test_net";
}
elsif ($file eq "db.ip6.V6NET")
{
my(@nibbles) = reverse(split /\s*/, $parm_ipv6_test_net);
$" = '.';
$outfile = "db.ip6.@nibbles";
$" = ' ';
}
print ">>Copying $basedir-src/$file to $basedir/$outfile\n" if $debug;
open(IN, "$parm_cwd/$basedir-src/$file") ||
tests_exit(-1, "Failed to open $parm_cwd/$basedir-src/$file: $!");
open(OUT, ">$parm_cwd/$basedir/$outfile") ||
tests_exit(-1, "Failed to open $parm_cwd/$basedir/$outfile: $!");
while (<IN>)
{
do_substitute(0);
print OUT;
}
close(IN);
close(OUT);
}
}
##################################################
# Create fake DNS zones for this host #
##################################################
# There are fixed zone files for 127.0.0.1 and ::1, but we also want to be
# sure that there are forward and reverse registrations for this host, using
# its real IP addresses. Dynamically created zone files achieve this.
if ($have_ipv4 || $have_ipv6)
{
my($shortname,$domain) = $parm_hostname =~ /^([^.]+)(.*)/;
open(OUT, ">$parm_cwd/dnszones/db$domain") ||
tests_exit(-1, "Failed to open $parm_cwd/dnszones/db$domain: $!");
print OUT "; This is a dynamically constructed fake zone file.\n" .
"; The following line causes fakens to return PASS_ON\n" .
"; for queries that it cannot answer\n\n" .
"PASS ON NOT FOUND\n\n";
print OUT "$shortname A $parm_ipv4\n" if $have_ipv4;
print OUT "$shortname AAAA $parm_ipv6\n" if $have_ipv6;
print OUT "\n; End\n";
close(OUT);
}
if ($have_ipv4 && $parm_ipv4 ne "127.0.0.1")
{
my(@components) = $parm_ipv4 =~ /^(\d+)\.(\d+)\.(\d+)\.(\d+)/;
open(OUT, ">$parm_cwd/dnszones/db.ip4.$components[0]") ||
tests_exit(-1,
"Failed to open $parm_cwd/dnszones/db.ip4.$components[0]: $!");
print OUT "; This is a dynamically constructed fake zone file.\n" .
"; The zone is $components[0].in-addr.arpa.\n\n" .
"$components[3].$components[2].$components[1] PTR $parm_hostname.\n\n" .
"; End\n";
close(OUT);
}
if ($have_ipv6 && $parm_ipv6 ne "::1")
{
my(@components) = split /:/, $parm_ipv6;
my(@nibbles) = reverse (split /\s*/, shift @components);
my($sep) = "";
$" = ".";
open(OUT, ">$parm_cwd/dnszones/db.ip6.@nibbles") ||
tests_exit(-1,
"Failed to open $parm_cwd/dnszones/db.ip6.@nibbles: $!");
print OUT "; This is a dynamically constructed fake zone file.\n" .
"; The zone is @nibbles.ip6.arpa.\n\n";
@components = reverse @components;
foreach $c (@components)
{
$c = "0$c" until $c =~ /^..../;
@nibbles = reverse(split /\s*/, $c);
print OUT "$sep@nibbles";
$sep = ".";
}
print OUT " PTR $parm_hostname.\n\n; End\n";
close(OUT);
$" = " ";
}
##################################################
# Create lists of mailboxes and message logs #
##################################################
# We use these lists to check that a test has created the expected files. It
# should be faster than looking for the file each time. For mailboxes, we have
# to scan a complete subtree, in order to handle maildirs. For msglogs, there
# is just a flat list of files.
@oldmails = list_files_below("mail");
opendir(DIR, "msglog") || tests_exit(-1, "Failed to opendir msglog: $!");
@oldmsglogs = readdir(DIR);
closedir(DIR);
##################################################
# Run the required tests #
##################################################
# Each test script contains a number of tests, separated by a line that
# contains ****. We open input from the terminal so that we can read responses
# to prompts.
open(T, "/dev/tty") || tests_exit(-1, "Failed to open /dev/tty: $!");
print "\nPress RETURN to run the tests: ";
$_ = <T>;
print "\n";
$lasttestdir = "";
foreach $test (@test_list)
{
local($lineno) = 0;
local($commandno) = 0;
local($subtestno) = 0;
local($testno) = substr($test, -4);
local($sortlog) = 0;
my($gnutls) = 0;
my($docheck) = 1;
my($thistestdir) = substr($test, 0, -5);
if ($lasttestdir ne $thistestdir)
{
$gnutls = 0;
if (-s "scripts/$thistestdir/REQUIRES")
{
my($indent) = "";
print "\n>>> The following tests require: ";
open(IN, "scripts/$thistestdir/REQUIRES") ||
tests_exit(-1, "Failed to open scripts/$thistestdir/REQUIRES: $1");
while (<IN>)
{
$gnutls = 1 if /^support GnuTLS/;
print $indent, $_;
$indent = ">>> ";
}
close(IN);
}
}
$lasttestdir = $thistestdir;
# Remove any debris in the spool directory and the test-mail directory
# and also the files for collecting stdout and stderr. Then put back
# the test-mail directory for appendfile deliveries.
system "sudo /bin/rm -rf spool test-*";
system "mkdir test-mail 2>/dev/null";
# A privileged Exim will normally make its own spool directory, but some of
# the tests run in unprivileged modes that don't always work if the spool
# directory isn't already there. What is more, we want anybody to be able
# to read it in order to find the daemon's pid.
system "mkdir spool; " .
"sudo chown $parm_eximuser:$parm_eximgroup spool; " .
"sudo chmod 0755 spool";
# Empty the cache that keeps track of things like message id mappings, and
# set up the initial sequence strings.
undef %cache;
$next_msgid = "aX";
$next_port = 1111;
$message_skip = 0;
$msglog_skip = 0;
$stderr_skip = 0;
$stdout_skip = 0;
$rmfiltertest = 0;
$is_ipv6test = 0;
# Remove the associative arrays used to hold checked mail files and msglogs
undef %expected_mails;
undef %expected_msglogs;
# Open the test's script
open(SCRIPT, "scripts/$test") ||
tests_exit(-1, "Failed to open \"scripts/$test\": $!");
# The first line in the script must be a comment that is used to identify
# the set of tests as a whole.
$_ = <SCRIPT>;
$lineno++;
tests_exit(-1, "Missing identifying comment at start of $test") if (!/^#/);
printf("%s %s", (substr $test, 5), (substr $_, 2));
# Loop for each of the subtests within the script. The variable $server_pid
# is used to remember the pid of a "server" process, for which we do not
# wait until we have waited for a subsequent command.
local($server_pid) = 0;
for ($commandno = 1; !eof SCRIPT; $commandno++)
{
# Skip further leading comments and blank lines, handle the flag setting
# commands, and deal with tests for IP support.
while (<SCRIPT>)
{
$lineno++;
if (/^no_message_check/) { $message_skip = 1; next; }
if (/^no_msglog_check/) { $msglog_skip = 1; next; }
if (/^no_stderr_check/) { $stderr_skip = 1; next; }
if (/^no_stdout_check/) { $stdout_skip = 1; next; }
if (/^rmfiltertest/) { $rmfiltertest = 1; next; }
if (/^sortlog/) { $sortlog = 1; next; }
if (/^need_ipv4/)
{
next if $have_ipv4;
print ">>> IPv4 is needed for test $testno, but is not available: skipping\n";
$docheck = 0; # don't check output
undef $_; # pretend EOF
last;
}
if (/^need_ipv6/)
{
if ($have_ipv6)
{
$is_ipv6test = 1;
next;
}
print ">>> IPv6 is needed for test $testno, but is not available: skipping\n";
$docheck = 0; # don't check output
undef $_; # pretend EOF
last;
}
if (/^need_move_frozen_messages/)
{
next if defined $parm_support{"move_frozen_messages"};
print ">>> move frozen message support is needed for test $testno, " .
"but is not\n>>> available: skipping\n";
$docheck = 0; # don't check output
undef $_; # pretend EOF
last;
}
last unless /^(#|\s*$)/;
}
last if !defined $_; # Hit EOF
my($subtest_startline) = $lineno;
# Now run the command. The function returns 0 if exim was run and waited
# for, 1 if any other command was run and waited for, and 2 if a command
# was run and not waited for (usually a daemon or server startup).
my($commandname) = "";
my($expectrc) = 0;
my($rc) = run_command($testno, \$subtestno, \$expectrc, \$commandname);
my($cmdrc) = $?;
print ">> rc=$rc cmdrc=$cmdrc\n" if $debug;
# Hit EOF after an initial return code number
tests_exit(-1, "Unexpected EOF in script") if ($rc == 4);
# Carry on with the next command if we did not wait for this one. $rc == 0
# if no subprocess was run; $rc == 3 if we started a process but did not
# wait for it.
next if ($rc == 0 || $rc == 3);
# We ran and waited for a command. Check for the expected result unless
# it died.
if ($cmdrc != $expectrc && !$sigpipehappened)
{
printf("** Command $commandno (\"$commandname\", starting at line $subtest_startline)\n");
if (($cmdrc & 0xff) == 0)
{
printf("** Return code %d (expected %d)", $cmdrc/256, $expectrc/256);
}
elsif (($cmdrc & 0xff00) == 0)
{ printf("** Killed by signal %d", $cmdrc & 255); }
else
{ printf("** Status %x", $cmdrc); }
for (;;)
{
print "\nshow stdErr, show stdOut, Continue (without file comparison), or Quit? [Q] ";
$_ = <T>;
tests_exit(1) if /^q?$/i;
last if /^c$/i;
if (/^e$/i)
{
system("$more test-stderr");
}
elsif (/^o$/i)
{
system("$more test-stdout");
}
}
$docheck = 0;
}
# If the command was exim, and a listening server is running, we can now
# close its input, which causes us to wait for it to finish, which is why
# we didn't close it earlier.
if ($rc == 2 && $server_pid != 0)
{
close SERVERCMD;
$server_pid = 0;
if ($? != 0)
{
if (($? & 0xff) == 0)
{ printf("Server return code %d", $?/256); }
elsif (($? & 0xff00) == 0)
{ printf("Server killed by signal %d", $? & 255); }
else
{ printf("Server status %x", $?); }
for (;;)
{
print "\nShow server stdout, Continue, or Quit? [Q] ";
$_ = <T>;
tests_exit(1) if /^q?$/i;
last if /^c$/i;
if (/^s$/i)
{
open(S, "test-stdout-server") ||
tests_exit(-1, "Failed to open test-stdout-server: $!");
print while <S>;
close(S);
}
}
}
}
}
close SCRIPT;
# The script has finished. Check the all the output that was generated. The
# function returns 0 if all is well, 1 if we should rerun the test (the files
# have been updated). It does not return if the user responds Q to a prompt.
if ($docheck)
{
if (check_output() != 0)
{
print (("#" x 79) . "\n");
redo;
}
else
{
print (" Script completed\n");
}
}
}
##################################################
# Exit from the test script #
##################################################
tests_exit(-1, "No runnable tests selected") if @test_list == 0;
tests_exit(0);
# End of runtest script
