benchmark_unittest.cc 5.42 KB
Newer Older
JINMEI Tatuya's avatar
JINMEI Tatuya committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14
// Copyright (C) 2010  Internet Systems Consortium, Inc. ("ISC")
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH
// REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
// AND FITNESS.  IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT,
// INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
// LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
// PERFORMANCE OF THIS SOFTWARE.

15
#include <time.h>               // for nanosleep
JINMEI Tatuya's avatar
JINMEI Tatuya committed
16 17 18 19 20 21 22 23 24 25 26 27 28

#include <bench/benchmark.h>

#include <gtest/gtest.h>

using namespace std;
using namespace isc::bench;

namespace {
// Our "benchmark" simply sleeps for a short period, and reports a faked
// number of iterations.
class TestBenchMark {
public:
29 30
    TestBenchMark(const int sub_iterations,
                  const struct timespec& sleep_time) :
JINMEI Tatuya's avatar
JINMEI Tatuya committed
31 32 33 34
        sub_iterations_(sub_iterations), sleep_time_(sleep_time),
        setup_completed_(false), teardown_completed_(false)
    {}
    unsigned int run() {
35
        nanosleep(&sleep_time_, NULL);
JINMEI Tatuya's avatar
JINMEI Tatuya committed
36 37 38
        return (sub_iterations_);
    }
    const int sub_iterations_;
39
    const struct timespec sleep_time_;
JINMEI Tatuya's avatar
JINMEI Tatuya committed
40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57
    bool setup_completed_;
    bool teardown_completed_;
};
}

namespace isc {
namespace bench {
template <>
void
BenchMark<TestBenchMark>::setUp() {
    target_.setup_completed_ = true;
};

template <>
void
BenchMark<TestBenchMark>::tearDown() {
    target_.teardown_completed_ = true;
};
58 59

// XXX: some compilers cannot find class static constants used in
Jeremy C. Reed's avatar
Jeremy C. Reed committed
60
// EXPECT_xxx macros, for which we need an explicit definition.
61
template <typename T>
62
const int BenchMark<T>::TIME_FAILURE;
JINMEI Tatuya's avatar
JINMEI Tatuya committed
63 64 65 66 67 68 69 70
}
}

namespace {
TEST(BenchMarkTest, run) {
    // use some uncommon iterations for testing purpose:
    const int sub_iterations = 23;
    const int sleep_time = 50000; // will sleep for 50ms
71
    const struct timespec sleep_timespec = { 0, sleep_time * 1000 };
JINMEI Tatuya's avatar
JINMEI Tatuya committed
72
    // we cannot expect particular accuracy on the measured duration, so
73
    // we'll include some conservative margin (50%) and perform range
JINMEI Tatuya's avatar
JINMEI Tatuya committed
74
    // comparison below.
75
    const int duration_margin = 25000; // 25ms
JINMEI Tatuya's avatar
JINMEI Tatuya committed
76 77
    const int ONE_MILLION = 1000000;

78 79
    // Prerequisite check: since the tests in this case may depend on subtle
    // timing, it may result in false positives.  There are reportedly systems
80
    // where sleeping doesn't work as this test expects.  So we check the
81 82
    // conditions before the tests, and if it fails skip the tests at the
    // risk of overlooking possible bugs.
83 84
    // We do this with a tighter margin than the checks themselves
    const int duration_soft_margin = 12500; // 12.5ms
85 86
    struct timeval check_begin, check_end;
    gettimeofday(&check_begin, NULL);
87
    nanosleep(&sleep_timespec, 0);
88 89 90 91 92 93 94 95 96 97
    gettimeofday(&check_end, NULL);
    check_end.tv_sec -= check_begin.tv_sec;
    if (check_end.tv_usec >= check_begin.tv_usec) {
        check_end.tv_usec = check_end.tv_usec - check_begin.tv_usec;
    } else {
        check_end.tv_usec = ONE_MILLION + check_begin.tv_usec -
            check_end.tv_usec;
        --check_end.tv_sec;
    }
    if (check_end.tv_sec != 0 ||
98 99
        sleep_time - duration_soft_margin > check_end.tv_usec ||
        sleep_time + duration_soft_margin < check_end.tv_usec) {
100
        cerr << "Prerequisite check failed.  skipping test" << endl;
JINMEI Tatuya's avatar
JINMEI Tatuya committed
101
        return;
102 103
    }

104
    TestBenchMark test_bench(sub_iterations, sleep_timespec);
JINMEI Tatuya's avatar
JINMEI Tatuya committed
105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136
    BenchMark<TestBenchMark> bench(1, test_bench, false);
    // Check pre-test conditions.
    EXPECT_FALSE(test_bench.setup_completed_);
    EXPECT_FALSE(test_bench.teardown_completed_);

    bench.run();

    // Check if specialized setup and teardown were performed.
    EXPECT_TRUE(test_bench.setup_completed_);
    EXPECT_TRUE(test_bench.teardown_completed_);

    // Check accuracy of the measured statistics.
    EXPECT_EQ(sub_iterations, bench.getIteration());
    EXPECT_LT(sleep_time - duration_margin, bench.getDuration() * ONE_MILLION);
    EXPECT_GT(sleep_time + duration_margin, bench.getDuration() * ONE_MILLION);
    EXPECT_LT((sleep_time - duration_margin) /
              static_cast<double>(sub_iterations),
              bench.getAverageTime() * ONE_MILLION);
    EXPECT_GT((sleep_time + duration_margin) /
              static_cast<double>(sub_iterations),
              bench.getAverageTime() * ONE_MILLION);
    EXPECT_LT(static_cast<double>(sub_iterations) /
              (sleep_time + duration_margin),
              bench.getIterationPerSecond() / ONE_MILLION);
    EXPECT_GT(static_cast<double>(sub_iterations) /
              (sleep_time - duration_margin),
              bench.getIterationPerSecond() / ONE_MILLION);
}

TEST(BenchMarkTest, runWithNoIteration) {
    // we'll lie on the number of iteration (0).  it will result in
    // meaningless result, but at least it shouldn't crash.
137 138
    const struct timespec null_timespec = { 0, 0 };
    TestBenchMark test_bench(0, null_timespec);
JINMEI Tatuya's avatar
JINMEI Tatuya committed
139 140 141 142 143 144 145 146
    BenchMark<TestBenchMark> bench(1, test_bench, false);
    bench.run();
    EXPECT_EQ(0, bench.getIteration());
    // Since the reported iteration is 0, naive calculation of the average
    // time would cause a division by 0 failure.
    EXPECT_EQ(bench.TIME_FAILURE, bench.getAverageTime());
}
}