Index: uspace/app/perf/Makefile =================================================================== --- uspace/app/perf/Makefile (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/perf/Makefile (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -34,4 +34,5 @@ SOURCES = \ + benchlist.c \ perf.c \ ipc/ns_ping.c \ Index: uspace/app/perf/benchlist.c =================================================================== --- uspace/app/perf/benchlist.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) +++ uspace/app/perf/benchlist.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2018 Jiri Svoboda + * Copyright (c) 2018 Vojtech Horky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup perf + * @{ + */ +/** + * @file + */ + +#include +#include "benchlist.h" + +benchmark_t *benchmarks[] = { + &bench_malloc1, + &bench_malloc2, + &bench_ns_ping, + &bench_ping_pong +}; + +size_t benchmark_count = sizeof(benchmarks) / sizeof(benchmarks[0]); + +/** @} + */ Index: uspace/app/perf/benchlist.h =================================================================== --- uspace/app/perf/benchlist.h (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) +++ uspace/app/perf/benchlist.h (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2018 Jiri Svoboda + * Copyright (c) 2019 Vojtech Horky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup perf + * @{ + */ +/** @file + */ + +#ifndef BENCHLIST_H_ +#define BENCHLIST_H_ + +#include "perf.h" + +extern benchmark_t bench_malloc1; +extern benchmark_t bench_malloc2; +extern benchmark_t bench_ns_ping; +extern benchmark_t bench_ping_pong; + +extern benchmark_t *benchmarks[]; +extern size_t benchmark_count; + +#endif + +/** @} + */ Index: uspace/app/perf/ipc/ns_ping.c =================================================================== --- uspace/app/perf/ipc/ns_ping.c (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/perf/ipc/ns_ping.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -27,108 +27,38 @@ */ -#include #include -#include -#include #include #include #include +#include +#include "../benchlist.h" #include "../perf.h" -#define MIN_DURATION_SECS 10 -#define NUM_SAMPLES 10 +static bool runner(stopwatch_t *stopwatch, uint64_t niter, + char *error, size_t error_size) +{ + stopwatch_start(stopwatch); -static errno_t ping_pong_measure(uint64_t niter, uint64_t *rduration) -{ - struct timespec start; - uint64_t count; + for (uint64_t count = 0; count < niter; count++) { + errno_t rc = ns_ping(); - getuptime(&start); - - for (count = 0; count < niter; count++) { - errno_t retval = ns_ping(); - - if (retval != EOK) { - printf("Error sending ping message.\n"); - return EIO; + if (rc != EOK) { + snprintf(error, error_size, + "failed sending ping message: %s (%d)", + str_error(rc), rc); + return false; } } - struct timespec now; - getuptime(&now); + stopwatch_stop(stopwatch); - *rduration = ts_sub_diff(&now, &start) / 1000; - return EOK; + return true; } -static void ping_pong_report(uint64_t niter, uint64_t duration) -{ - printf("Completed %" PRIu64 " round trips in %" PRIu64 " us", - niter, duration); - - if (duration > 0) { - printf(", %" PRIu64 " rt/s.\n", niter * 1000 * 1000 / duration); - } else { - printf(".\n"); - } -} - -const char *bench_ns_ping(void) -{ - errno_t rc; - uint64_t duration; - uint64_t dsmp[NUM_SAMPLES]; - - printf("Warm up and determine work size...\n"); - - struct timespec start; - getuptime(&start); - - uint64_t niter = 1; - - while (true) { - rc = ping_pong_measure(niter, &duration); - if (rc != EOK) - return "Failed."; - - ping_pong_report(niter, duration); - - if (duration >= MIN_DURATION_SECS * 1000000) - break; - - niter *= 2; - } - - printf("Measure %d samples...\n", NUM_SAMPLES); - - int i; - - for (i = 0; i < NUM_SAMPLES; i++) { - rc = ping_pong_measure(niter, &dsmp[i]); - if (rc != EOK) - return "Failed."; - - ping_pong_report(niter, dsmp[i]); - } - - double sum = 0.0; - - for (i = 0; i < NUM_SAMPLES; i++) - sum += (double)niter / ((double)dsmp[i] / 1000000.0l); - - double avg = sum / NUM_SAMPLES; - - double qd = 0.0; - double d; - for (i = 0; i < NUM_SAMPLES; i++) { - d = (double)niter / ((double)dsmp[i] / 1000000.0l) - avg; - qd += d * d; - } - - double stddev = qd / (NUM_SAMPLES - 1); // XXX sqrt - - printf("Average: %.0f rt/s Std.dev^2: %.0f rt/s Samples: %d\n", - avg, stddev, NUM_SAMPLES); - - return NULL; -} +benchmark_t bench_ns_ping = { + .name = "ns_ping", + .desc = "Name service IPC ping-pong benchmark", + .entry = &runner, + .setup = NULL, + .teardown = NULL +}; Index: uspace/app/perf/ipc/ns_ping.def =================================================================== --- uspace/app/perf/ipc/ns_ping.def (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ (revision ) @@ -1,5 +1,0 @@ -{ - "ns_ping", - "Name service IPC ping-pong benchmark", - &bench_ns_ping -}, Index: uspace/app/perf/ipc/ping_pong.c =================================================================== --- uspace/app/perf/ipc/ping_pong.c (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/perf/ipc/ping_pong.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -27,123 +27,59 @@ */ -#include #include -#include -#include #include #include #include +#include +#include "../benchlist.h" #include "../perf.h" -#define MIN_DURATION_SECS 10 -#define NUM_SAMPLES 10 +static ipc_test_t *test = NULL; -static errno_t ping_pong_measure(ipc_test_t *test, uint64_t niter, - uint64_t *rduration) +static bool setup(char *error, size_t error_size) { - struct timespec start; - uint64_t count; + errno_t rc = ipc_test_create(&test); + if (rc != EOK) { + snprintf(error, error_size, + "failed contacting IPC test server: %s (%d)", + str_error(rc), rc); + return false; + } - getuptime(&start); + return true; +} - for (count = 0; count < niter; count++) { - errno_t retval = ipc_test_ping(test); +static bool teardown(char *error, size_t error_size) +{ + ipc_test_destroy(test); + return true; +} - if (retval != EOK) { - printf("Error sending ping message.\n"); - return EIO; +static bool runner(stopwatch_t *stopwatch, uint64_t niter, + char *error, size_t error_size) +{ + stopwatch_start(stopwatch); + + for (uint64_t count = 0; count < niter; count++) { + errno_t rc = ipc_test_ping(test); + + if (rc != EOK) { + snprintf(error, error_size, + "failed sending ping message: %s (%d)", + str_error(rc), rc); + return false; } } - struct timespec now; - getuptime(&now); + stopwatch_stop(stopwatch); - *rduration = ts_sub_diff(&now, &start) / 1000; - return EOK; + return true; } -static void ping_pong_report(uint64_t niter, uint64_t duration) -{ - printf("Completed %" PRIu64 " round trips in %" PRIu64 " us", - niter, duration); - - if (duration > 0) { - printf(", %" PRIu64 " rt/s.\n", niter * 1000 * 1000 / duration); - } else { - printf(".\n"); - } -} - -const char *bench_ping_pong(void) -{ - errno_t rc; - uint64_t duration; - uint64_t dsmp[NUM_SAMPLES]; - ipc_test_t *test; - const char *msg; - - rc = ipc_test_create(&test); - if (rc != EOK) - return "Failed contacting IPC test server."; - - printf("Warm up and determine work size...\n"); - - struct timespec start; - getuptime(&start); - - uint64_t niter = 1; - - while (true) { - rc = ping_pong_measure(test, niter, &duration); - if (rc != EOK) { - msg = "Failed."; - goto error; - } - - ping_pong_report(niter, duration); - - if (duration >= MIN_DURATION_SECS * 1000000) - break; - - niter *= 2; - } - - printf("Measure %d samples...\n", NUM_SAMPLES); - - int i; - - for (i = 0; i < NUM_SAMPLES; i++) { - rc = ping_pong_measure(test, niter, &dsmp[i]); - if (rc != EOK) { - msg = "Failed."; - goto error; - } - - ping_pong_report(niter, dsmp[i]); - } - - double sum = 0.0; - - for (i = 0; i < NUM_SAMPLES; i++) - sum += (double)niter / ((double)dsmp[i] / 1000000.0l); - - double avg = sum / NUM_SAMPLES; - - double qd = 0.0; - double d; - for (i = 0; i < NUM_SAMPLES; i++) { - d = (double)niter / ((double)dsmp[i] / 1000000.0l) - avg; - qd += d * d; - } - - double stddev = qd / (NUM_SAMPLES - 1); // XXX sqrt - - printf("Average: %.0f rt/s Std.dev^2: %.0f rt/s Samples: %d\n", - avg, stddev, NUM_SAMPLES); - - ipc_test_destroy(test); - return NULL; -error: - ipc_test_destroy(test); - return msg; -} +benchmark_t bench_ping_pong = { + .name = "ping_pong", + .desc = "IPC ping-pong benchmark", + .entry = &runner, + .setup = &setup, + .teardown = &teardown +}; Index: uspace/app/perf/ipc/ping_pong.def =================================================================== --- uspace/app/perf/ipc/ping_pong.def (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ (revision ) @@ -1,5 +1,0 @@ -{ - "ping_pong", - "IPC ping-pong benchmark", - &bench_ping_pong -}, Index: uspace/app/perf/malloc/malloc1.c =================================================================== --- uspace/app/perf/malloc/malloc1.c (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/perf/malloc/malloc1.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -30,109 +30,31 @@ #include #include -#include -#include +#include "../benchlist.h" #include "../perf.h" -#define MIN_DURATION_SECS 10 -#define NUM_SAMPLES 10 - -static errno_t malloc1_measure(uint64_t niter, uint64_t *rduration) +static bool runner(stopwatch_t *stopwatch, uint64_t size, + char *error, size_t error_size) { - struct timespec start; - uint64_t count; - void *p; - - getuptime(&start); - - for (count = 0; count < niter; count++) { - p = malloc(1); - if (p == NULL) - return ENOMEM; + stopwatch_start(stopwatch); + for (uint64_t i = 0; i < size; i++) { + void *p = malloc(1); + if (p == NULL) { + snprintf(error, error_size, + "failed to allocate 1B in run %" PRIu64 " (out of %" PRIu64 ")", + i, size); + return false; + } free(p); } + stopwatch_stop(stopwatch); - struct timespec now; - getuptime(&now); - - *rduration = ts_sub_diff(&now, &start) / 1000; - return EOK; + return true; } -static void malloc1_report(uint64_t niter, uint64_t duration) -{ - printf("Completed %" PRIu64 " allocations and deallocations in %" PRIu64 " us", - niter, duration); - - if (duration > 0) { - printf(", %" PRIu64 " cycles/s.\n", niter * 1000 * 1000 / duration); - } else { - printf(".\n"); - } -} - -const char *bench_malloc1(void) -{ - errno_t rc; - uint64_t duration; - uint64_t dsmp[NUM_SAMPLES]; - const char *msg; - - printf("Warm up and determine work size...\n"); - - struct timespec start; - getuptime(&start); - - uint64_t niter = 1; - - while (true) { - rc = malloc1_measure(niter, &duration); - if (rc != EOK) { - msg = "Failed."; - goto error; - } - - malloc1_report(niter, duration); - - if (duration >= MIN_DURATION_SECS * 1000000) - break; - - niter *= 2; - } - - printf("Measure %d samples...\n", NUM_SAMPLES); - - int i; - - for (i = 0; i < NUM_SAMPLES; i++) { - rc = malloc1_measure(niter, &dsmp[i]); - if (rc != EOK) { - msg = "Failed."; - goto error; - } - - malloc1_report(niter, dsmp[i]); - } - - double sum = 0.0; - - for (i = 0; i < NUM_SAMPLES; i++) - sum += (double)niter / ((double)dsmp[i] / 1000000.0l); - - double avg = sum / NUM_SAMPLES; - - double qd = 0.0; - double d; - for (i = 0; i < NUM_SAMPLES; i++) { - d = (double)niter / ((double)dsmp[i] / 1000000.0l) - avg; - qd += d * d; - } - - double stddev = qd / (NUM_SAMPLES - 1); // XXX sqrt - - printf("Average: %.0f cycles/s Std.dev^2: %.0f cycles/s Samples: %d\n", - avg, stddev, NUM_SAMPLES); - - return NULL; -error: - return msg; -} +benchmark_t bench_malloc1 = { + .name = "malloc1", + .desc = "User-space memory allocator benchmark, repeatedly allocate one block", + .entry = &runner, + .setup = NULL, + .teardown = NULL +}; Index: uspace/app/perf/malloc/malloc1.def =================================================================== --- uspace/app/perf/malloc/malloc1.def (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ (revision ) @@ -1,5 +1,0 @@ -{ - "malloc1", - "User-space memory allocator benchmark, repeatedly allocate one block", - &bench_malloc1 -}, Index: uspace/app/perf/malloc/malloc2.c =================================================================== --- uspace/app/perf/malloc/malloc2.c (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/perf/malloc/malloc2.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -27,120 +27,51 @@ */ -#include +#include #include -#include -#include -#include +#include "../benchlist.h" #include "../perf.h" -#define MIN_DURATION_SECS 10 -#define NUM_SAMPLES 10 +static bool runner(stopwatch_t *stopwatch, uint64_t niter, + char *error, size_t error_size) +{ + stopwatch_start(stopwatch); -static errno_t malloc2_measure(uint64_t niter, uint64_t *rduration) -{ - struct timespec start; - uint64_t count; - void **p; - - getuptime(&start); - - p = malloc(niter * sizeof(void *)); - if (p == NULL) - return ENOMEM; - - for (count = 0; count < niter; count++) { - p[count] = malloc(1); - if (p[count] == NULL) - return ENOMEM; + void **p = malloc(niter * sizeof(void *)); + if (p == NULL) { + snprintf(error, error_size, + "failed to allocate backend array (%" PRIu64 "B)", + niter * sizeof(void *)); + return false; } - for (count = 0; count < niter; count++) + for (uint64_t count = 0; count < niter; count++) { + p[count] = malloc(1); + if (p[count] == NULL) { + snprintf(error, error_size, + "failed to allocate 1B in run %" PRIu64 " (out of %" PRIu64 ")", + count, niter); + for (uint64_t j = 0; j < count; j++) { + free(p[j]); + } + free(p); + return false; + } + } + + for (uint64_t count = 0; count < niter; count++) free(p[count]); free(p); - struct timespec now; - getuptime(&now); + stopwatch_stop(stopwatch); - *rduration = ts_sub_diff(&now, &start) / 1000; - return EOK; + return true; } -static void malloc2_report(uint64_t niter, uint64_t duration) -{ - printf("Completed %" PRIu64 " allocations and deallocations in %" PRIu64 " us", - niter, duration); - - if (duration > 0) { - printf(", %" PRIu64 " cycles/s.\n", niter * 1000 * 1000 / duration); - } else { - printf(".\n"); - } -} - -const char *bench_malloc2(void) -{ - errno_t rc; - uint64_t duration; - uint64_t dsmp[NUM_SAMPLES]; - const char *msg; - - printf("Warm up and determine work size...\n"); - - struct timespec start; - getuptime(&start); - - uint64_t niter = 1; - - while (true) { - rc = malloc2_measure(niter, &duration); - if (rc != EOK) { - msg = "Failed."; - goto error; - } - - malloc2_report(niter, duration); - - if (duration >= MIN_DURATION_SECS * 1000000) - break; - - niter *= 2; - } - - printf("Measure %d samples...\n", NUM_SAMPLES); - - int i; - - for (i = 0; i < NUM_SAMPLES; i++) { - rc = malloc2_measure(niter, &dsmp[i]); - if (rc != EOK) { - msg = "Failed."; - goto error; - } - - malloc2_report(niter, dsmp[i]); - } - - double sum = 0.0; - - for (i = 0; i < NUM_SAMPLES; i++) - sum += (double)niter / ((double)dsmp[i] / 1000000.0l); - - double avg = sum / NUM_SAMPLES; - - double qd = 0.0; - double d; - for (i = 0; i < NUM_SAMPLES; i++) { - d = (double)niter / ((double)dsmp[i] / 1000000.0l) - avg; - qd += d * d; - } - - double stddev = qd / (NUM_SAMPLES - 1); // XXX sqrt - - printf("Average: %.0f cycles/s Std.dev^2: %.0f cycles/s Samples: %d\n", - avg, stddev, NUM_SAMPLES); - - return NULL; -error: - return msg; -} +benchmark_t bench_malloc2 = { + .name = "malloc2", + .desc = "User-space memory allocator benchmark, allocate many small blocks", + .entry = &runner, + .setup = NULL, + .teardown = NULL +}; Index: uspace/app/perf/malloc/malloc2.def =================================================================== --- uspace/app/perf/malloc/malloc2.def (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ (revision ) @@ -1,5 +1,0 @@ -{ - "malloc2", - "User-space memory allocator benchmark, allocate many small blocks", - &bench_malloc2 -}, Index: uspace/app/perf/perf.c =================================================================== --- uspace/app/perf/perf.c (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/perf/perf.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -1,4 +1,5 @@ /* * Copyright (c) 2018 Jiri Svoboda + * Copyright (c) 2018 Vojtech Horky * All rights reserved. * @@ -34,37 +35,204 @@ */ +#include +#include #include #include #include #include +#include +#include +#include +#include #include "perf.h" - -benchmark_t benchmarks[] = { -#include "ipc/ns_ping.def" -#include "ipc/ping_pong.def" -#include "malloc/malloc1.def" -#include "malloc/malloc2.def" - { NULL, NULL, NULL } -}; +#include "benchlist.h" + +#define MIN_DURATION_SECS 10 +#define NUM_SAMPLES 10 +#define MAX_ERROR_STR_LENGTH 1024 + +static void short_report(stopwatch_t *stopwatch, int run_index, + benchmark_t *bench, uint64_t workload_size) +{ + usec_t duration_usec = NSEC2USEC(stopwatch_get_nanos(stopwatch)); + + printf("Completed %" PRIu64 " operations in %llu us", + workload_size, duration_usec); + if (duration_usec > 0) { + double nanos = stopwatch_get_nanos(stopwatch); + double thruput = (double) workload_size / (nanos / 1000000000.0l); + printf(", %.0f ops/s.\n", thruput); + } else { + printf(".\n"); + } +} + +/* + * This is a temporary solution until we have proper sqrt() implementation + * in libmath. + * + * The algorithm uses Babylonian method [1]. + * + * [1] https://en.wikipedia.org/wiki/Methods_of_computing_square_roots#Babylonian_method + */ +static double estimate_square_root(double value, double precision) +{ + double estimate = 1.; + double prev_estimate = estimate + 10 * precision; + + while (fabs(estimate - prev_estimate) > precision) { + prev_estimate = estimate; + estimate = (prev_estimate + value / prev_estimate) / 2.; + } + + return estimate; +} + +/* + * Compute available statistics from given stopwatches. + * + * We compute normal mean for average duration of the workload and geometric + * mean for average thruput. Note that geometric mean is necessary to compute + * average throughput correctly - consider the following example: + * - we run always 60 operations, + * - first run executes in 30 s (i.e. 2 ops/s) + * - and second one in 10 s (6 ops/s). + * Then, naively, average throughput would be (2+6)/2 = 4 [ops/s]. However, we + * actually executed 60 + 60 ops in 30 + 10 seconds. So the actual average + * throughput is 3 ops/s (which is exactly what geometric mean means). + * + */ +static void compute_stats(stopwatch_t *stopwatch, size_t stopwatch_count, + uint64_t workload_size, double precision, double *out_duration_avg, + double *out_duration_sigma, double *out_thruput_avg) +{ + double inv_thruput_sum = 0.0; + double nanos_sum = 0.0; + double nanos_sum2 = 0.0; + + for (size_t i = 0; i < stopwatch_count; i++) { + double nanos = stopwatch_get_nanos(&stopwatch[i]); + double thruput = (double) workload_size / nanos; + + inv_thruput_sum += 1.0 / thruput; + nanos_sum += nanos; + nanos_sum2 += nanos * nanos; + } + *out_duration_avg = nanos_sum / stopwatch_count; + double sigma2 = (nanos_sum2 - nanos_sum * (*out_duration_avg)) / + ((double) stopwatch_count - 1); + // FIXME: implement sqrt properly + *out_duration_sigma = estimate_square_root(sigma2, precision); + *out_thruput_avg = 1.0 / (inv_thruput_sum / stopwatch_count); +} + +static void summary_stats(stopwatch_t *stopwatch, size_t stopwatch_count, + benchmark_t *bench, uint64_t workload_size) +{ + double duration_avg, duration_sigma, thruput_avg; + compute_stats(stopwatch, stopwatch_count, workload_size, 0.001, + &duration_avg, &duration_sigma, &thruput_avg); + + printf("Average: %" PRIu64 " ops in %.0f us (sd %.0f us); " + "%.0f ops/s; Samples: %zu\n", + workload_size, duration_avg / 1000.0, duration_sigma / 1000.0, + thruput_avg * 1000000000.0, stopwatch_count); +} static bool run_benchmark(benchmark_t *bench) { - /* Execute the benchmarl */ - const char *ret = bench->entry(); - - if (ret == NULL) { - printf("\nBenchmark completed\n"); - return true; - } - - printf("\n%s\n", ret); - return false; + printf("Warm up and determine workload size...\n"); + + char *error_msg = malloc(MAX_ERROR_STR_LENGTH + 1); + if (error_msg == NULL) { + printf("Out of memory!\n"); + return false; + } + str_cpy(error_msg, MAX_ERROR_STR_LENGTH, ""); + + bool ret = true; + + if (bench->setup != NULL) { + ret = bench->setup(error_msg, MAX_ERROR_STR_LENGTH); + if (!ret) { + goto leave_error; + } + } + + /* + * Find workload size that is big enough to last few seconds. + * We also check that uint64_t is big enough. + */ + uint64_t workload_size = 0; + for (size_t bits = 0; bits <= 64; bits++) { + if (bits == 64) { + str_cpy(error_msg, MAX_ERROR_STR_LENGTH, "Workload too small even for 1 << 63"); + goto leave_error; + } + workload_size = ((uint64_t) 1) << bits; + + stopwatch_t stopwatch = STOPWATCH_INITIALIZE_STATIC; + + bool ok = bench->entry(&stopwatch, workload_size, + error_msg, MAX_ERROR_STR_LENGTH); + if (!ok) { + goto leave_error; + } + short_report(&stopwatch, -1, bench, workload_size); + + nsec_t duration = stopwatch_get_nanos(&stopwatch); + if (duration > SEC2NSEC(MIN_DURATION_SECS)) { + break; + } + } + + printf("Workload size set to %" PRIu64 ", measuring %d samples.\n", workload_size, NUM_SAMPLES); + + stopwatch_t *stopwatch = calloc(NUM_SAMPLES, sizeof(stopwatch_t)); + if (stopwatch == NULL) { + snprintf(error_msg, MAX_ERROR_STR_LENGTH, "failed allocating memory"); + goto leave_error; + } + for (int i = 0; i < NUM_SAMPLES; i++) { + stopwatch_init(&stopwatch[i]); + + bool ok = bench->entry(&stopwatch[i], workload_size, + error_msg, MAX_ERROR_STR_LENGTH); + if (!ok) { + free(stopwatch); + goto leave_error; + } + short_report(&stopwatch[i], i, bench, workload_size); + } + + summary_stats(stopwatch, NUM_SAMPLES, bench, workload_size); + printf("\nBenchmark completed\n"); + + free(stopwatch); + + goto leave; + +leave_error: + printf("Error: %s\n", error_msg); + ret = false; + +leave: + if (bench->teardown != NULL) { + bool ok = bench->teardown(error_msg, MAX_ERROR_STR_LENGTH); + if (!ok) { + printf("Error: %s\n", error_msg); + ret = false; + } + } + + free(error_msg); + + return ret; } static int run_benchmarks(void) { - benchmark_t *bench; - unsigned int i = 0; - unsigned int n = 0; + unsigned int count_ok = 0; + unsigned int count_fail = 0; char *failed_names = NULL; @@ -72,16 +240,16 @@ printf("\n*** Running all benchmarks ***\n\n"); - for (bench = benchmarks; bench->name != NULL; bench++) { - printf("%s (%s)\n", bench->name, bench->desc); - if (run_benchmark(bench)) { - i++; + for (size_t it = 0; it < benchmark_count; it++) { + printf("%s (%s)\n", benchmarks[it]->name, benchmarks[it]->desc); + if (run_benchmark(benchmarks[it])) { + count_ok++; continue; } if (!failed_names) { - failed_names = str_dup(bench->name); + failed_names = str_dup(benchmarks[it]->name); } else { char *f = NULL; - asprintf(&f, "%s, %s", failed_names, bench->name); + asprintf(&f, "%s, %s", failed_names, benchmarks[it]->name); if (!f) { printf("Out of memory.\n"); @@ -91,12 +259,13 @@ failed_names = f; } - n++; - } - - printf("\nCompleted, %u benchmarks run, %u succeeded.\n", i + n, i); + count_fail++; + } + + printf("\nCompleted, %u benchmarks run, %u succeeded.\n", + count_ok + count_fail, count_ok); if (failed_names) printf("Failed benchmarks: %s\n", failed_names); - return n; + return count_fail; } @@ -104,20 +273,16 @@ { size_t len = 0; - benchmark_t *bench; - for (bench = benchmarks; bench->name != NULL; bench++) { - if (str_length(bench->name) > len) - len = str_length(bench->name); - } - - unsigned int _len = (unsigned int) len; - if ((_len != len) || (((int) _len) < 0)) { - printf("Command length overflow\n"); - return; - } - - for (bench = benchmarks; bench->name != NULL; bench++) - printf("%-*s %s\n", _len, bench->name, bench->desc); - - printf("%-*s Run all benchmarks\n", _len, "*"); + for (size_t i = 0; i < benchmark_count; i++) { + size_t len_now = str_length(benchmarks[i]->name); + if (len_now > len) + len = len_now; + } + + assert(can_cast_size_t_to_int(len) && "benchmark name length overflow"); + + for (size_t i = 0; i < benchmark_count; i++) + printf("%-*s %s\n", (int) len, benchmarks[i]->name, benchmarks[i]->desc); + + printf("%-*s Run all benchmarks\n", (int) len, "*"); } @@ -135,8 +300,7 @@ } - benchmark_t *bench; - for (bench = benchmarks; bench->name != NULL; bench++) { - if (str_cmp(argv[1], bench->name) == 0) { - return (run_benchmark(bench) ? 0 : -1); + for (size_t i = 0; i < benchmark_count; i++) { + if (str_cmp(argv[1], benchmarks[i]->name) == 0) { + return (run_benchmark(benchmarks[i]) ? 0 : -1); } } Index: uspace/app/perf/perf.h =================================================================== --- uspace/app/perf/perf.h (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/perf/perf.h (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -37,6 +37,9 @@ #include +#include -typedef const char *(*benchmark_entry_t)(void); +typedef bool (*benchmark_entry_t)(stopwatch_t *, uint64_t, + char *, size_t); +typedef bool (*benchmark_helper_t)(char *, size_t); typedef struct { @@ -44,12 +47,7 @@ const char *desc; benchmark_entry_t entry; + benchmark_helper_t setup; + benchmark_helper_t teardown; } benchmark_t; - -extern const char *bench_malloc1(void); -extern const char *bench_malloc2(void); -extern const char *bench_ns_ping(void); -extern const char *bench_ping_pong(void); - -extern benchmark_t benchmarks[]; #endif Index: uspace/app/tester/tester.c =================================================================== --- uspace/app/tester/tester.c (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/app/tester/tester.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -35,4 +35,5 @@ */ +#include #include #include @@ -40,4 +41,5 @@ #include #include +#include #include "tester.h" @@ -144,15 +146,11 @@ } - unsigned int _len = (unsigned int) len; - if ((_len != len) || (((int) _len) < 0)) { - printf("Command length overflow\n"); - return; - } + assert(can_cast_size_t_to_int(len) && "test name length overflow"); for (test = tests; test->name != NULL; test++) - printf("%-*s %s%s\n", _len, test->name, test->desc, + printf("%-*s %s%s\n", (int) len, test->name, test->desc, (test->safe ? "" : " (unsafe)")); - printf("%-*s Run all safe tests\n", _len, "*"); + printf("%-*s Run all safe tests\n", (int) len, "*"); } Index: uspace/lib/c/Makefile =================================================================== --- uspace/lib/c/Makefile (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/lib/c/Makefile (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -189,4 +189,5 @@ TEST_SOURCES = \ test/adt/circ_buf.c \ + test/casting.c \ test/fibril/timer.c \ test/main.c \ @@ -196,4 +197,5 @@ test/stdio/scanf.c \ test/odict.c \ + test/perf.c \ test/perm.c \ test/qsort.c \ Index: uspace/lib/c/include/perf.h =================================================================== --- uspace/lib/c/include/perf.h (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) +++ uspace/lib/c/include/perf.h (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -0,0 +1,105 @@ +/* + * Copyright (c) 2018 Vojtech Horky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup libc + * @{ + */ +/** @file + * System performance measurement utilities. + */ + +#ifndef LIBC_PERF_H_ +#define LIBC_PERF_H_ + +#include + +/** Stopwatch is THE way to measure elapsed time on HelenOS. */ +typedef struct { + struct timespec start; + struct timespec end; +} stopwatch_t; + +#define STOPWATCH_INITIALIZE_STATIC { { 0, 0 }, { 0, 0 } } + +/** Initialize system stopwatch. + * + * @param stopwatch Stopwatch to initialize. + */ +static inline void stopwatch_init(stopwatch_t *stopwatch) +{ + stopwatch->start.tv_sec = 0; + stopwatch->start.tv_nsec = 0; + stopwatch->end.tv_sec = 0; + stopwatch->end.tv_nsec = 0; +} + +/** Emulate elapsed time for use in tests. + * + * @param stopwatch Stopwatch to use. + * @param nanos Elapsed time in nanoseconds to set. + */ +static inline void stopwatch_set_elapsed(stopwatch_t *stopwatch, nsec_t nanos) +{ + stopwatch->start.tv_sec = 0; + stopwatch->start.tv_nsec = 0; + stopwatch->end.tv_sec = NSEC2SEC(nanos); + stopwatch->end.tv_nsec = nanos - SEC2NSEC(stopwatch->end.tv_sec); +} + +/** Start the stopwatch. + * + * @param stopwatch Stopwatch to start. + */ +static inline void stopwatch_start(stopwatch_t *stopwatch) +{ + getuptime(&stopwatch->start); +} + +/** Stop the stopwatch. + * + * @param stopwatch Stopwatch to stop. + */ +static inline void stopwatch_stop(stopwatch_t *stopwatch) +{ + getuptime(&stopwatch->end); +} + +/** Get elapsed time in nanoseconds. + * + * @param stopwatch Stopwatch to read from. + * @return Elapsed time in nanoseconds. + */ +static inline nsec_t stopwatch_get_nanos(stopwatch_t *stopwatch) +{ + return ts_sub_diff(&stopwatch->end, &stopwatch->start); +} + +#endif + +/** @} + */ Index: uspace/lib/c/include/types/casting.h =================================================================== --- uspace/lib/c/include/types/casting.h (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) +++ uspace/lib/c/include/types/casting.h (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019 Vojtech Horky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +/** @addtogroup libc + * @{ + */ +/** @file + */ + +#ifndef LIBC_TYPES_CASTING_H_ +#define LIBC_TYPES_CASTING_H_ + +#include +#include + +/** + * Checks that the size_t value can be casted to int without loss of data. + * + * @param val Value to test. + * @return Whether it is safe to typecast value to int. + */ +static inline bool can_cast_size_t_to_int(size_t val) +{ + unsigned int as_uint = (unsigned int) val; + return !((as_uint != val) || (((int) as_uint) < 0)); +} + +#endif + +/** @} + */ Index: uspace/lib/c/test/casting.c =================================================================== --- uspace/lib/c/test/casting.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) +++ uspace/lib/c/test/casting.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2019 Vojtech Horky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +PCUT_INIT; + +PCUT_TEST_SUITE(casting); + +/* + * Following tests checks functionality of can_cast_size_t_to_int. + */ + +PCUT_TEST(size_t_to_int_with_small) +{ + PCUT_ASSERT_TRUE(can_cast_size_t_to_int(0)); + PCUT_ASSERT_TRUE(can_cast_size_t_to_int(128)); +} + +PCUT_TEST(size_t_to_int_with_biggest_int) +{ + PCUT_ASSERT_TRUE(can_cast_size_t_to_int(INT_MAX)); +} + +PCUT_TEST(size_t_to_int_with_biggest_size_t) +{ + PCUT_ASSERT_FALSE(can_cast_size_t_to_int(SIZE_MAX)); +} + +PCUT_EXPORT(casting); Index: uspace/lib/c/test/main.c =================================================================== --- uspace/lib/c/test/main.c (revision 7acd787ef6db58d28e222cd14600149ac8c9ff72) +++ uspace/lib/c/test/main.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -32,4 +32,5 @@ PCUT_INIT; +PCUT_IMPORT(casting); PCUT_IMPORT(circ_buf); PCUT_IMPORT(fibril_timer); @@ -37,4 +38,5 @@ PCUT_IMPORT(mem); PCUT_IMPORT(odict); +PCUT_IMPORT(perf); PCUT_IMPORT(perm); PCUT_IMPORT(qsort); Index: uspace/lib/c/test/perf.c =================================================================== --- uspace/lib/c/test/perf.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) +++ uspace/lib/c/test/perf.c (revision b9f1585fca6387ad56c5d7cef8501f54f3f3dfc1) @@ -0,0 +1,80 @@ +/* + * Copyright (c) 2018 Vojtech Horky + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * - The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include + +PCUT_INIT; + +PCUT_TEST_SUITE(perf); + +/* Checks that initialization zeroes out all entries. */ +PCUT_TEST(zero_diff) +{ + stopwatch_t sw; + stopwatch_init(&sw); + PCUT_ASSERT_INT_EQUALS(0, (int) stopwatch_get_nanos(&sw)); +} + +/* Checks that initialization zeroes out all entries. */ +PCUT_TEST(zero_diff_static) +{ + stopwatch_t sw = STOPWATCH_INITIALIZE_STATIC; + PCUT_ASSERT_INT_EQUALS(0, (int) stopwatch_get_nanos(&sw)); +} + +/* Checks that measuring 1s sleep does not give completely invalid results. */ +PCUT_TEST(stopwatch_smokes) +{ + stopwatch_t sw = STOPWATCH_INITIALIZE_STATIC; + stopwatch_start(&sw); + fibril_sleep(1); + stopwatch_stop(&sw); + nsec_t diff_nanos = stopwatch_get_nanos(&sw); + PCUT_ASSERT_TRUE(diff_nanos > MSEC2NSEC(500)); + PCUT_ASSERT_TRUE(diff_nanos < SEC2NSEC(5)); +} + +/* Checks that setting time works for small values. */ +PCUT_TEST(stopwatch_emulation_works_small) +{ + stopwatch_t sw = STOPWATCH_INITIALIZE_STATIC; + stopwatch_set_elapsed(&sw, 42); + PCUT_ASSERT_INT_EQUALS(42, (int) stopwatch_get_nanos(&sw)); +} + +/* Checks that setting time works for big values too. */ +PCUT_TEST(stopwatch_emulation_works_big) +{ + stopwatch_t sw = STOPWATCH_INITIALIZE_STATIC; + stopwatch_set_elapsed(&sw, 4200000000021); + PCUT_ASSERT_EQUALS(4200000000021, (long long) stopwatch_get_nanos(&sw)); +} + +PCUT_EXPORT(perf);