+-------------------------------------------+
+Item hash list
+~~~~~~~~~~~~~~
+
+To reduce the number of reads performed from flash memory, each member of Page class maintains a list of pairs: (item index; item hash). This list makes searches much quicker. Instead of iterating over all entries, reading them from flash one at a time, ``Page::findItem`` first performs search for item hash in the hash list. This gives the item index within the page, if such an item exists. Due to a hash collision it is possible that a different item will be found. This is handled by falling back to iteration over items in flash.
+
+Each node in hash list contains a 24-bit hash and 8-bit item index. Hash is calculated based on item namespace and key name. CRC32 is used for calculation, result is truncated to 24 bits. To reduce overhead of storing 32-bit entries in a linked list, list is implemented as a doubly-linked list of arrays. Each array holds 29 entries, for the total size of 128 bytes, together with linked list pointers and 32-bit count field. Minimal amount of extra RAM useage per page is therefore 128 bytes, maximum is 640 bytes.
+
+
--- /dev/null
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "nvs_item_hash_list.hpp"
+
+namespace nvs
+{
+
+HashList::~HashList()
+{
+ for (auto it = mBlockList.begin(); it != mBlockList.end();) {
+ auto tmp = it;
+ ++it;
+ mBlockList.erase(tmp);
+ delete static_cast<HashListBlock*>(tmp);
+ }
+}
+
+HashList::HashListBlock::HashListBlock()
+{
+ static_assert(sizeof(HashListBlock) == HashListBlock::BYTE_SIZE,
+ "cache block size calculation incorrect");
+}
+
+void HashList::insert(const Item& item, size_t index)
+{
+ const uint32_t hash_24 = item.calculateCrc32WithoutValue() & 0xffffff;
+ // add entry to the end of last block if possible
+ if (mBlockList.size()) {
+ auto& block = mBlockList.back();
+ if (block.mCount < HashListBlock::ENTRY_COUNT) {
+ block.mNodes[block.mCount++] = HashListNode(hash_24, index);
+ return;
+ }
+ }
+ // if the above failed, create a new block and add entry to it
+ HashListBlock* newBlock = new HashListBlock;
+ mBlockList.push_back(newBlock);
+ newBlock->mNodes[0] = HashListNode(hash_24, index);
+ newBlock->mCount++;
+}
+
+void HashList::erase(size_t index)
+{
+ for (auto it = std::begin(mBlockList); it != std::end(mBlockList);)
+ {
+ bool haveEntries = false;
+ for (size_t i = 0; i < it->mCount; ++i) {
+ if (it->mNodes[i].mIndex == index) {
+ it->mNodes[i].mIndex = 0xff;
+ return;
+ }
+ if (it->mNodes[i].mIndex != 0xff) {
+ haveEntries = true;
+ }
+ }
+ if (!haveEntries) {
+ auto tmp = it;
+ ++it;
+ mBlockList.erase(tmp);
+ delete static_cast<HashListBlock*>(tmp);
+ }
+ else {
+ ++it;
+ }
+ }
+ assert(false && "item should have been present in cache");
+}
+
+size_t HashList::find(size_t start, const Item& item)
+{
+ const uint32_t hash_24 = item.calculateCrc32WithoutValue() & 0xffffff;
+ for (auto it = std::begin(mBlockList); it != std::end(mBlockList); ++it)
+ {
+ for (size_t index = 0; index < it->mCount; ++index) {
+ HashListNode& e = it->mNodes[index];
+ if (e.mIndex >= start &&
+ e.mHash == hash_24 &&
+ e.mIndex != 0xff) {
+ return e.mIndex;
+ }
+ }
+ }
+ return SIZE_MAX;
+}
+
+
+} // namespace nvs
--- /dev/null
+// Copyright 2015-2016 Espressif Systems (Shanghai) PTE LTD
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef nvs_item_hash_list_h
+#define nvs_item_hash_list_h
+
+#include "nvs.h"
+#include "nvs_types.hpp"
+#include "intrusive_list.h"
+
+namespace nvs
+{
+
+class HashList
+{
+public:
+ ~HashList();
+ void insert(const Item& item, size_t index);
+ void erase(const size_t index);
+ size_t find(size_t start, const Item& item);
+
+protected:
+
+ struct HashListNode {
+ HashListNode() :
+ mIndex(0xff), mHash(0)
+ {
+ }
+
+ HashListNode(uint32_t hash, size_t index) :
+ mIndex((uint32_t) index), mHash(hash)
+ {
+ }
+
+ uint32_t mIndex : 8;
+ uint32_t mHash : 24;
+ };
+
+ struct HashListBlock : public intrusive_list_node<HashList::HashListBlock>
+ {
+ HashListBlock();
+
+ static const size_t BYTE_SIZE = 128;
+ static const size_t ENTRY_COUNT = (BYTE_SIZE - sizeof(intrusive_list_node<HashListBlock>) - sizeof(size_t)) / 4;
+
+ size_t mCount = 0;
+ HashListNode mNodes[ENTRY_COUNT];
+ };
+
+
+ typedef intrusive_list<HashListBlock> TBlockList;
+ TBlockList mBlockList;
+}; // class HashList
+
+} // namespace nvs
+
+
+#endif /* nvs_item_hash_list_h */
// write first item
- item.nsIndex = nsIndex;
- item.datatype = datatype;
- item.span = (totalSize + ENTRY_SIZE - 1) / ENTRY_SIZE;
- item.reserved = 0xff;
-
- std::fill_n(reinterpret_cast<uint32_t*>(item.key), sizeof(item.key) / 4, 0xffffffff);
- std::fill_n(reinterpret_cast<uint32_t*>(item.data), sizeof(item.data) / 4, 0xffffffff);
-
- strncpy(item.key, key, sizeof(item.key) - 1);
- item.key[sizeof(item.key) - 1] = 0;
-
+ size_t span = (totalSize + ENTRY_SIZE - 1) / ENTRY_SIZE;
+ item = Item(nsIndex, datatype, span, key);
+ mHashList.insert(item, mNextFreeEntry);
+
if (datatype != ItemType::SZ && datatype != ItemType::BLOB) {
memcpy(item.data, data, dataSize);
item.crc32 = item.calculateCrc32();
{
auto state = mEntryTable.get(index);
assert(state == EntryState::WRITTEN || state == EntryState::EMPTY);
-
+ mHashList.erase(index);
+
size_t span = 1;
if (state == EntryState::WRITTEN) {
Item item;
if (err != ESP_OK) {
return err;
}
+ other.mHashList.insert(entry, other.mNextFreeEntry);
err = other.writeEntry(entry);
if (err != ESP_OK) {
return err;
}
continue;
}
+
+ mHashList.insert(item, i);
if (item.datatype != ItemType::BLOB && item.datatype != ItemType::SZ) {
continue;
}
}
}
+ else if (mState == PageState::FULL || mState == PageState::FREEING) {
+ // We have already filled mHashList for page in active state.
+ // Do the same for the case when page is in full or freeing state.
+ Item item;
+ for (size_t i = mFirstUsedEntry; i < ENTRY_COUNT; ++i) {
+ if (mEntryTable.get(i) != EntryState::WRITTEN) {
+ continue;
+ }
+
+ auto err = readEntry(i, item);
+ if (err != ESP_OK) {
+ mState = PageState::INVALID;
+ return err;
+ }
+
+ mHashList.insert(item, i);
+
+ size_t span = item.span;
+ i += span - 1;
+ }
+
+ }
return ESP_OK;
}
if (end > ENTRY_COUNT) {
end = ENTRY_COUNT;
}
-
+
+ if (nsIndex != NS_ANY && datatype != ItemType::ANY && key != NULL) {
+ size_t cachedIndex = mHashList.find(start, Item(nsIndex, datatype, 0, key));
+ if (cachedIndex < ENTRY_COUNT) {
+ start = cachedIndex;
+ }
+ else {
+ return ESP_ERR_NVS_NOT_FOUND;
+ }
+ }
+
size_t next;
for (size_t i = start; i < end; i = next) {
next = i + 1;
mFirstUsedEntry = INVALID_ENTRY;
mNextFreeEntry = INVALID_ENTRY;
mState = PageState::UNINITIALIZED;
+ mHashList = HashList();
return ESP_OK;
}
#include "esp_spi_flash.h"
#include "compressed_enum_table.hpp"
#include "intrusive_list.h"
+#include "nvs_item_hash_list.hpp"
namespace nvs
{
uint16_t mErasedEntryCount = 0;
CachedFindInfo mFindInfo;
+ HashList mHashList;
static const uint32_t HEADER_OFFSET = 0;
static const uint32_t ENTRY_TABLE_OFFSET = HEADER_OFFSET + 32;
namespace nvs
{
-uint32_t Item::calculateCrc32()
+uint32_t Item::calculateCrc32() const
{
uint32_t result = 0xffffffff;
const uint8_t* p = reinterpret_cast<const uint8_t*>(this);
return result;
}
+uint32_t Item::calculateCrc32WithoutValue() const
+{
+ uint32_t result = 0xffffffff;
+ const uint8_t* p = reinterpret_cast<const uint8_t*>(this);
+ result = crc32_le(result, p + offsetof(Item, nsIndex),
+ offsetof(Item, datatype) - offsetof(Item, nsIndex));
+ result = crc32_le(result, p + offsetof(Item, key), sizeof(key));
+ return result;
+}
+
uint32_t Item::calculateCrc32(const uint8_t* data, size_t size)
{
uint32_t result = 0xffffffff;
};
static const size_t MAX_KEY_LENGTH = sizeof(key) - 1;
+
+ Item(uint8_t nsIndex, ItemType datatype, uint8_t span, const char* key_)
+ : nsIndex(nsIndex), datatype(datatype), span(span), reserved(0xff)
+ {
+ std::fill_n(reinterpret_cast<uint32_t*>(key), sizeof(key) / 4, 0xffffffff);
+ std::fill_n(reinterpret_cast<uint32_t*>(data), sizeof(data) / 4, 0xffffffff);
+ if (key_) {
+ strncpy(key, key_, sizeof(key) - 1);
+ key[sizeof(key) - 1] = 0;
+ } else {
+ key[0] = 0;
+ }
+ }
+
+ Item()
+ {
+ }
- uint32_t calculateCrc32();
+ uint32_t calculateCrc32() const;
+ uint32_t calculateCrc32WithoutValue() const;
static uint32_t calculateCrc32(const uint8_t* data, size_t size);
void getKey(char* dst, size_t dstSize)
nvs_page.cpp \
nvs_pagemanager.cpp \
nvs_storage.cpp \
+ nvs_item_hash_list.cpp \
) \
spi_flash_emulation.cpp \
test_compressed_enum_table.cpp \
projects / esp-idf / commitdiff