From: Alexander A. Klimov Date: Tue, 5 Jun 2018 13:01:43 +0000 (+0200) Subject: Implement object packer for consistent hashing X-Git-Tag: v2.10.0~82^2 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=dd8cb429694bc77d4363f37307ca9e42c869fbce;p=icinga2 Implement object packer for consistent hashing --- diff --git a/lib/base/CMakeLists.txt b/lib/base/CMakeLists.txt index 29daa94a0..c02001ccc 100644 --- a/lib/base/CMakeLists.txt +++ b/lib/base/CMakeLists.txt @@ -57,6 +57,7 @@ set(base_SOURCES number.cpp number.hpp number-script.cpp object.cpp object.hpp object-script.cpp objectlock.cpp objectlock.hpp + object-packer.cpp object-packer.hpp objecttype.cpp objecttype.hpp perfdatavalue.cpp perfdatavalue.hpp perfdatavalue-ti.hpp primitivetype.cpp primitivetype.hpp diff --git a/lib/base/object-packer.cpp b/lib/base/object-packer.cpp new file mode 100644 index 000000000..988504fe7 --- /dev/null +++ b/lib/base/object-packer.cpp @@ -0,0 +1,268 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#include "base/object-packer.hpp" +#include "base/debug.hpp" +#include "base/dictionary.hpp" +#include "base/array.hpp" +#include "base/objectlock.hpp" +#include +#include +#include +#include +#include + +using namespace icinga; + +// Just for the sake of code readability +typedef std::vector StringBuilder; + +union EndiannessDetector +{ + EndiannessDetector() + { + i = 1; + } + + int i; + char buf[sizeof(int)]; +}; + +static const EndiannessDetector l_EndiannessDetector; + +// Assumption: The compiler will optimize (away) if/else statements using this. +#define MACHINE_LITTLE_ENDIAN (l_EndiannessDetector.buf[0]) + +static void PackAny(const Value& value, StringBuilder& builder); + +/** + * std::swap() seems not to work + */ +static inline void SwapBytes(char& a, char& b) +{ + char c = a; + a = b; + b = c; +} + +#if CHAR_MIN != 0 +union CharU2SConverter +{ + CharU2SConverter() + { + s = 0; + } + + unsigned char u; + signed char s; +}; +#endif + +/** + * Avoid implementation-defined overflows during unsigned to signed casts + */ +static inline char UIntToByte(unsigned i) +{ +#if CHAR_MIN == 0 + return i; +#else + CharU2SConverter converter; + + converter.u = i; + return converter.s; +#endif +} + +/** + * Append the given int as big-endian 64-bit unsigned int + */ +static inline void PackUInt64BE(uint_least64_t i, StringBuilder& builder) +{ + char buf[8] = { + UIntToByte(i >> 56u), + UIntToByte((i >> 48u) & 255u), + UIntToByte((i >> 40u) & 255u), + UIntToByte((i >> 32u) & 255u), + UIntToByte((i >> 24u) & 255u), + UIntToByte((i >> 16u) & 255u), + UIntToByte((i >> 8u) & 255u), + UIntToByte(i & 255u) + }; + + builder.insert(builder.end(), (char*)buf, (char*)buf + 8); +} + +union Double2BytesConverter +{ + Double2BytesConverter() + { + buf[0] = 0; + buf[1] = 0; + buf[2] = 0; + buf[3] = 0; + buf[4] = 0; + buf[5] = 0; + buf[6] = 0; + buf[7] = 0; + } + + double f; + char buf[8]; +}; + +/** + * Append the given double as big-endian IEEE 754 binary64 + */ +static inline void PackFloat64BE(double f, StringBuilder& builder) +{ + Double2BytesConverter converter; + + converter.f = f; + + if (MACHINE_LITTLE_ENDIAN) { + SwapBytes(converter.buf[0], converter.buf[7]); + SwapBytes(converter.buf[1], converter.buf[6]); + SwapBytes(converter.buf[2], converter.buf[5]); + SwapBytes(converter.buf[3], converter.buf[4]); + } + + builder.insert(builder.end(), (char*)converter.buf, (char*)converter.buf + 8); +} + +/** + * Append the given string's length (BE uint64) and the string itself + */ +static inline void PackString(const String& string, StringBuilder& builder) +{ + PackUInt64BE(string.GetLength(), builder); + builder.insert(builder.end(), string.Begin(), string.End()); +} + +/** + * Append the given array + */ +static inline void PackArray(const Array::Ptr& arr, StringBuilder& builder) +{ + ObjectLock olock(arr); + + builder.emplace_back(5); + PackUInt64BE(arr->GetLength(), builder); + + for (const Value& value : arr) { + PackAny(value, builder); + } +} + +/** + * Append the given dictionary + */ +static inline void PackDictionary(const Dictionary::Ptr& dict, StringBuilder& builder) +{ + ObjectLock olock(dict); + + builder.emplace_back(6); + PackUInt64BE(dict->GetLength(), builder); + + for (const Dictionary::Pair& kv : dict) { + PackString(kv.first, builder); + PackAny(kv.second, builder); + } +} + +/** + * Append any JSON-encodable value + */ +static void PackAny(const Value& value, StringBuilder& builder) +{ + switch (value.GetType()) { + case ValueString: + builder.emplace_back(4); + PackString(value.Get(), builder); + break; + + case ValueNumber: + builder.emplace_back(3); + PackFloat64BE(value.Get(), builder); + break; + + case ValueBoolean: + builder.emplace_back(value.ToBool() ? 2 : 1); + break; + + case ValueEmpty: + builder.emplace_back(0); + break; + + case ValueObject: + { + const Object::Ptr& obj = value.Get(); + + Dictionary::Ptr dict = dynamic_pointer_cast(obj); + if (dict) { + PackDictionary(dict, builder); + break; + } + + Array::Ptr arr = dynamic_pointer_cast(obj); + if (arr) { + PackArray(arr, builder); + break; + } + } + + builder.emplace_back(0); + break; + + default: + VERIFY(!"Invalid variant type."); + } +} + +/** + * Pack any JSON-encodable value to a BSON-similar structure suitable for consistent hashing + * + * Spec: + * null: 0x00 + * false: 0x01 + * true: 0x02 + * number: 0x03 (ieee754_binary64_bigendian)payload + * string: 0x04 (uint64_bigendian)payload.length (char[])payload + * array: 0x05 (uint64_bigendian)payload.length (any[])payload + * object: 0x06 (uint64_bigendian)payload.length (keyvalue[])payload.sort() + * + * any: null|false|true|number|string|array|object + * keyvalue: (uint64_bigendian)key.length (char[])key (any)value + * + * Assumptions: + * - double is IEEE 754 binary64 + * - all int types (signed and unsigned) and all float types share the same endianness + * - char is exactly 8 bits wide and one char is exactly one byte affected by the machine endianness + * - all input strings, arrays and dictionaries are at most 2^64-1 long + * + * If not, this function will silently produce invalid results. + */ +String icinga::PackObject(const Value& value) +{ + StringBuilder builder; + PackAny(value, builder); + + String result; + result.insert(result.End(), builder.begin(), builder.end()); + return std::move(result); +} diff --git a/lib/base/object-packer.hpp b/lib/base/object-packer.hpp new file mode 100644 index 000000000..420a43979 --- /dev/null +++ b/lib/base/object-packer.hpp @@ -0,0 +1,35 @@ +/****************************************************************************** + * Icinga 2 * + * Copyright (C) 2012-2018 Icinga Development Team (https://www.icinga.com/) * + * * + * This program is free software; you can redistribute it and/or * + * modify it under the terms of the GNU General Public License * + * as published by the Free Software Foundation; either version 2 * + * of the License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU General Public License for more details. * + * * + * You should have received a copy of the GNU General Public License * + * along with this program; if not, write to the Free Software Foundation * + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA. * + ******************************************************************************/ + +#ifndef OBJECT_PACKER +#define OBJECT_PACKER + +#include "base/i2-base.hpp" + +namespace icinga +{ + +class String; +class Value; + +String PackObject(const Value& value); + +} + +#endif /* OBJECT_PACKER */