Unit testing in C: a tutorial with AcuTest, part II

We continue to explore unit testing in C, using the unit test framework as a prototypical example. We focus on output and test control flow this time.

Checks with messages via TEST_CHECK_

We begin with the following simple example program:

#include <math.h>
#include "acutest.h"

static double my_sqrt_float( double d )
{
    // perform four iterations of the Heron method to approximation the sqrt of d
    float x = d;
    for( int i = 0; i < 4; i++ )
    {
        x = ( x + d / x ) / 2.;
    }
    return x;
}

static double my_sqrt_double( double d )
{
    double x = d;
    // as above, but with double precision
    for( int i = 0; i < 4; i++ )
    {
        x = ( x + d / x ) / 2.;
    }
    return x;
}

void heron_with_float(void)
{
    float S = 2.;
    float expected = sqrt( S );
    float received = my_sqrt_float( S );
    
    TEST_CHECK_( fabs( expected-received ) < 1e-5, "sqrt of %e: expected: %e received: %e", (double)S, expected, received );
}

void heron_with_double(void)
{
    for( int S = 1; S < 10; S++ )
    {
        double expected = sqrt( (double)S );
        double received = my_sqrt_double( (double)S );
        
        TEST_CHECK_( fabs( expected-received ) < 1e-5, "sqrt of %e: expected: %e received: %e", (double)S, expected, received );
    }
}

TEST_LIST = {
    { "sqrt-float-test", heron_with_float },
    { "sqrt-double-test", heron_with_double },
    { NULL, NULL }
};

Compile and run as follows, not forgetting to include the mathematics library:

gcc -std=c99 -Wall -Wextra -pedantic demo.c -o demo -lm
./demo

This demonstrates the usage of the TEST_CHECK_ macro. The TEST_CHECK_ macro works like the TEST_CHECK macro, but allows for a custom error message. The format of the error message is the same as for printf, and the arguments are passed in the same way, except that a newline is added automatically at the end of each output.

Notice that the error message is only evaluated if the test fails, so it does not affect the performance of the test when it passes.

We also notice that varyadic macros in C require at least one argument in the list of varyadic arguments, so we cannot use TEST_CHECK_ without at least the string argument for the error message, even if we do not want to use any format specifiers in the error message.

More complex output with TEST_MSG

If a test fails, we might want to emit a more complicated error message, or maybe we prefer to not put the error in the check for source code readability reasons. In that case, we can use TEST_MSG. TEST_MSG is a macro that takes a format string and arguments, similar to printf, and emits the message if the most recent check has failed. It can be used in combination with TEST_CHECK.

We inspect the following example.

#include <math.h>
#include "acutest.h"

static double my_sqrt_float( double d )
{
    // perform four iterations of the Heron method to approximation the sqrt of d
    float x = d;
    for( int i = 0; i < 4; i++ )
    {
        x = ( x + d / x ) / 2.;
    }
    return x;
}

static double my_sqrt_double( double d )
{
    double x = d;
    // as above, but with double precision
    for( int i = 0; i < 4; i++ )
    {
        x = ( x + d / x ) / 2.;
    }
    return x;
}

void heron_with_float(void)
{
    {
        float Sf = 2.f;
        float expected = sqrt( Sf );
        float received = my_sqrt_float( Sf );
        float absolute_error = fabs(received - expected);
        float relative_error = fabs(absolute_error / expected);
        float tolerance = 1e-5;

        TEST_CHECK( fabs( expected-received ) < tolerance );
        TEST_MSG("input:     %.17g", Sf);
        TEST_MSG("expected:  %.17g", expected);
        TEST_MSG("actual:    %.17g", received);
        TEST_MSG("abs error: %.17g", absolute_error);
        TEST_MSG("rel error: %.17g", relative_error);
        TEST_MSG("tolerance: %.17g", tolerance);
    }
}

void heron_with_double(void)
{
    for( int S = 1; S < 10; S++ )
    {
        double Sd = (double)S;
        double expected = sqrt( Sd );
        double received = my_sqrt_double( Sd );
        double absolute_error = fabs(received - expected);
        double relative_error = fabs(absolute_error / expected);
        double tolerance = 1e-5;

        TEST_CHECK( fabs( expected-received ) < tolerance );
        TEST_MSG("input:     %.17g", Sd);
        TEST_MSG("expected:  %.17g", expected);
        TEST_MSG("actual:    %.17g", received);
        TEST_MSG("abs error: %.17g", absolute_error);
        TEST_MSG("rel error: %.17g", relative_error);
        TEST_MSG("tolerance: %.17g", tolerance);
    }
}

TEST_LIST = {
    { "sqrt-float-test", heron_with_float },
    { "sqrt-double-test", heron_with_double },
    { NULL, NULL }
};

If the check is successful, then the diagnostic messages remain invisible. But if the check fails, then all subsequent TEST_MSG macros emit the messages.

Aborting the unit test: TEST_ASSERT and TEST_ASSERT_

A complex test may involve a build up of data structures, so that each step relies on the previous step. For example, if one step involves allocating memory, we want to check that the allocation was successful before proceeding to the next step, and abort the test altogether if the allocation failed.

This can be done with the macros TEST_ASSERT and TEST_ASSERT_. The following code demonstrates the usage of these macros.

#include <stdlib.h>
#include "acutest.h"

static double* make_vector( size_t n )
{
    double* x = malloc( n * sizeof(double) );

    if( x == NULL ) 
    {
        return NULL;
    }

    for( size_t i = 0; i < n; i++ ) 
    {
        x[i] = (double)i;
    }

    return NULL; // accidentally return NULL instead of x. In real life, this could be a typo.
}

static double sum_vector( const double* x, size_t n )
{
    double sum = 0.;

    for( size_t i = 0; i < n; i++ ) 
    {
        sum += x[i];
    }

    return sum;
}

void test_sum_vector(void)
{
    size_t n = 4;
    double* x = make_vector(n);

    TEST_ASSERT( x != NULL );

    TEST_CHECK( sum_vector(x, n) == 6. );

    free(x);
}

TEST_LIST = {
    { "sum-vector", test_sum_vector },
    { NULL, NULL }
};

If the allocation of the vector fails, the test will be aborted. Any other test may still run, but the test that relies on the allocation will stop and be marked as failed.

Thus, TEST_ASSERT is a harder version of TEST_CHECK. The macro TEST_ASSERT is used to check critical conditions that must be true for the test to proceed, while TEST_CHECK is used for conditions that can be checked but do not necessarily need to be true for the test to continue.

Please notice that TEST_ASSERT is not a general replacement for TEST_CHECK. It should be used, e.g., to ensure that critical preconditions are met for subsequent check, or if a check failure is so severe that further checks are considered pointless.

There is also TEST_ASSERT_, which allows you to provide a custom message that will be displayed if the assertion fails. It works completely analogous to TEST_CHECK_ but aborts the entire test on failure and has it marked as failure.
Since no further messages will be emitted in the same test after TEST_ASSERT, we can still provide some data for posteriority via TEST_ASSERT_. For example,

TEST_ASSERT_( x != NULL, "Failed to allocate a vector with %zu entries.", n );

The check-and-assert pattern

A useful pattern for more complicated messages is to first perform a check and then call the assertion with the same expression.

We may that TEST_CHECK and TEST_CHECK_ return whether the boolean expression succeeded or not. These macros expand to functions, whereas the TEST_ASSERT AND TEST_ASSERT_ macros expand to control statements without a value.

if( !TEST_CHECK_( x != NULL, "vector allocation failed" ) ) 
{
    TEST_MSG( "requested entries: %zu", n );
    TEST_MSG( "requested bytes:   %zu", n * sizeof(double) );
    TEST_ASSERT( x != NULL );
}

A fixture-like construct in C

Even though AcuTest, being a very light-weight framework, does not explicitly offer fixtures as a feature, we can easily emulate fixtures.

Fixture is a fixed test set-up that can be set up and torn down repeatedly for several tests.

#include <math.h>
#include <stddef.h>
#include <stdlib.h>
#include "acutest.h"

static double l2_norm(const double* x, size_t n)
{
    double sum = 0.;

    for( size_t i = 0; i < n; i++ ) 
    {
        sum += x[i] * x[i];
    }

    return sqrt(sum);
}

struct vector_fixture 
{
    double* x;
    size_t n;
};

static struct vector_fixture make_fixture(void)
{
    struct vector_fixture f;

    f.n = 2;
    f.x = malloc( f.n * sizeof(double) );

    TEST_ASSERT_( f.x != NULL, "allocate fixture vector" );

    f.x[0] = 3.;
    f.x[1] = 4.;

    return f;
}

static void destroy_fixture(struct vector_fixture* f)
{
    free(f->x);
    f->x = NULL;
    f->n = 0;
}

void test_fixture_norm(void)
{
    struct vector_fixture f = make_fixture();

    double norm = l2_norm( f.x, f.n );

    TEST_CHECK( norm == 5. );

    destroy_fixture(&f);
}

TEST_LIST = {
    { "test-fixture-norm", test_fixture_norm },
    { NULL, NULL }
};

Skipping tests

We may choose to skip a test for various reasons. Perhaps there is a feature not yet implemented, making the test pointless, or the test is so resource-intensive, we may decide to temporarily disable it. Or there is a condition that can only be checked dynamically, such as resource capacity of a system component.

The TEST_SKIP can be called from within a test and marks the test as skipped. It must be called before any check or assertion has been performed. The control flow of the function still proceeds and should return from the function without further invoking any checks (or assertions).