From 105900702410e8f2e8b3cecac5b19c6b7ca60bc5 Mon Sep 17 00:00:00 2001 From: Eric Haszlakiewicz Date: Mon, 27 Apr 2020 04:00:00 +0000 Subject: [PATCH] Add an initial version of a benchmarking harness for json-c, to be able to more easily compare the performance of different library versions. --- bench/README.bench.md | 80 +++++++++++++++ bench/jc-bench.sh | 220 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 300 insertions(+) create mode 100644 bench/README.bench.md create mode 100755 bench/jc-bench.sh diff --git a/bench/README.bench.md b/bench/README.bench.md new file mode 100644 index 0000000..62c6908 --- /dev/null +++ b/bench/README.bench.md @@ -0,0 +1,80 @@ + +Benchmark tests for json-c + +General strategy: +------------------- + +* Identify "after" commit hash + * Use provided directory + * Use provided commit hash + * Local changes in current working directory + * ${cur_branch} +* Identify "before" commit hash, in order of preference + * Use provided directory + * Use provided commit hash + * Use origin/${cur_branch}, if different from ${after_commit} + * Use previous release + +* If not using existing dir, clone to src-${after_commit} + * or, checkout appropriate commit in existing src-${after_commit} +* Create build & install dirs for ${after_commit} +* Build & install ${after_commit} +* Compile benchmark programs against install-${after_commit} + +* If not using existing dir, clone to src-${before_commit} + * or, checkout appropriate commit in existing src-${before_commit} +* Create build & install dirs for ${before_commit} +* Build & install ${before_commit} +* Compile benchmark programs against install-${before_commit} + +* Run benchmark in each location +* Compare results + +heaptrack memory profiler +--------------------------- + +https://milianw.de/blog/heaptrack-a-heap-memory-profiler-for-linux.html + + +``` +yum install libdwarf-devel elfutils boost-devel libunwind-devel + +git clone git://anongit.kde.org/heaptrack +cd heaptrack +mkdir build +cd build +cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_INSTALL_PREFIX=$HOME/heaptrack-install .. +make install +``` + + +Issues +-------- + +* jc-bench.sh is incomplete. + +* valgrind massif misreports "extra-heap" bytes? + + "json_parse -n canada.json" shows 38640 KB maxrss. + + Using valgrind --tool=massif, a large amount of memory is listed as + wasted "extra-heap" bytes. (~5.6MB) + + ``` + valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n ~/canada.json + ms_print massif.out + ``` + + + Using heaptrack, and analyzing the histogram, only shows ~2.6MB + ``` + heaptrack ./json_parse -n canada.json + heaptrack --analyze heaptrack*gz -H histgram.out + awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out + ``` + + With the (unreleased) arraylist trimming changes, maxrss reported by + getrusage() goes down, but massif claims *more* total usage, and a HUGE + extra-heap amount (50% of total). + diff --git a/bench/jc-bench.sh b/bench/jc-bench.sh new file mode 100755 index 0000000..6eea46f --- /dev/null +++ b/bench/jc-bench.sh @@ -0,0 +1,220 @@ +#!/bin/sh +# +# Benchmarking harness for json-c +# +# Use this to compare the behavior of two different versions of the library, +# such as json-c-0.14 release vs master, master vs local changes, etc... +# + +set -e + +trap 'echo "FAILED!"' EXIT + +RUNDIR=$(dirname "$0") +RUNDIR=$(cd "$RUNDIR" && pwd) + +TOP=$(cd "$RUNDIR/.." && pwd) + +usage() +{ + exitval=$1 + errmsg=$2 + if [ $exitval -ne 0 ] ; then + exec 1>&2 + fi + if [ ! -z "$errmsg" ] ; then + echo "ERROR: $errmsg" 1>&2 + fi + cat < /dev/null || true) + echo "at_commit: $at_commit" + if [ -z "$at_commit" ] ; then + # Assume it's an empty dir + git clone -n "$TOP" "$src_dir" + fi + git -C "$src_dir" --git-dir="$src_dir/.git" checkout "$src_commit" + fi + # else, use the provided $src_dir + + cd "${build_dir}" + cmake -DCMAKE_INSTALL_PREFIX="${inst_dir}" "${src_dir}" + make all install + + cd "${bench_dir}" + cmake -DCMAKE_PREFIX_PATH="${inst_dir}" "${TOP}/apps" + make all +} + +# XXX TODO: name "after" and "before" uniquely using the dir & commit + +if [ $do_all -ne 0 -o $do_build -ne 0 ] ; then + compile_benchmark "after" "${after_src_dir}" "${after_commit}" + compile_benchmark "before" "${before_src_dir}" "${before_commit}" +fi + +run_benchmark() +{ + local bname=$1 + local inst_dir="${WORK}/$bname/install" + local bench_dir="${WORK}/$bname/bench" + + local INPUT=${WORK}/../canada.json + + cd "${bench_dir}" + mkdir -p results + (time ./json_parse -n "${INPUT}") > results/basic_timing.out 2>&1 + valgrind --tool=massif --massif-out-file=massif.out ./json_parse -n "${INPUT}" + ms_print massif.out > results/ms_print.out + heaptrack -o heaptrack_out ./json_parse -n "${INPUT}" + heaptrack --analyze heaptrack_out.gz -H histogram.out > results/heaptrack.out + awk ' { s=$1; count=$2; ru=(int((s+ 15) / 16)) * 16; wasted = ((ru-s)*count); print s, count, ru-s, wasted; total=total+wasted} END { print "Total: ", total }' histogram.out > results/histogram2.out + + # XXX stamp some info about what was built & run into ./results/. + + echo "DONE with $bname" +} + +if [ $do_all -ne 0 -o $do_run -ne 0 ] ; then + run_benchmark "after" + run_benchmark "before" +fi + +if [ $do_compare -ne 0 ] ; then + # XXX this needs better analysis + cd "${WORK}" + diff -udr before/bench/results after/bench/results || true +else + echo "To compare results, run:" + echo "$0 --compare" +fi + +trap '' EXIT + +:<<=cut + +Benchmarks to run: + +* Parse JSON strings, of various sizes and characteristics + * Flags: STRICT vs. non-STRICT, validate UTF8 + +* Serialization time + * plain, spaces, pretty + +* json_c_visit tests +* JSON pointer tests + +Things to record and compare: + +* Running time +* Peak memory usage +* Useful bytes vs. overhead for memory allocations +* Total number of allocations +* Average allocation size +* Log of all allocation sizes + +=cut + -- 2.50.1