Add hash index to speed up searching

demos/linux/applications/fdb_cfg.h

@@ -25,11 +25,15 @@
 
 /* Using file storage mode by POSIX file API, like open/read/write/close */
 #define FDB_USING_FILE_POSIX_MODE
+//#define FDB_USING_FILE_LIBC_MODE
 
 /* log print macro. default EF_PRINT macro is printf() */
 /* #define FDB_PRINT(...)              my_printf(__VA_ARGS__) */
 
 /* print debug information */
 #define FDB_DEBUG_ENABLE
 
+/* enable hash enhancement, open it can speed up searhing */
+//#define FDB_KV_CACHE_HASH_ENHANCEMENT
+
 #endif /* _FDB_CFG_H_ */

demos/linux/applications/main.c

@@ -9,18 +9,23 @@
 #include <sys/stat.h>
 #include <sys/types.h>
 #include <flashdb.h>
+#include <time.h>
 
 #define FDB_LOG_TAG "[main]"
 
+#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT
+static struct fdb_cache_hash_enhancement_ops hash_ops;
+#endif
+
 static pthread_mutex_t kv_locker, ts_locker;
 static uint32_t boot_count = 0;
 static time_t boot_time[10] = {0, 1, 2, 3};
 /* default KV nodes */
 static struct fdb_default_kv_node default_kv_table[] = {
-        {"username", "armink", 0}, /* string KV */
-        {"password", "123456", 0}, /* string KV */
-        {"boot_count", &boot_count, sizeof(boot_count)}, /* int type KV */
-        {"boot_time", &boot_time, sizeof(boot_time)},    /* int array type KV */
+    {"username", "armink", 0},                       /* string KV */
+    {"password", "123456", 0},                       /* string KV */
+    {"boot_count", &boot_count, sizeof(boot_count)}, /* int type KV */
+    {"boot_time", &boot_time, sizeof(boot_time)},    /* int array type KV */
 };
 /* KVDB object */
 static struct fdb_kvdb kvdb = { 0 };
@@ -32,6 +37,7 @@ static int counts = 0;
 extern void kvdb_basic_sample(fdb_kvdb_t kvdb);
 extern void kvdb_type_string_sample(fdb_kvdb_t kvdb);
 extern void kvdb_type_blob_sample(fdb_kvdb_t kvdb);
+extern void kvdb_bench_sample(fdb_kvdb_t kvdb);
 extern void tsdb_sample(fdb_tsdb_t tsdb);
 
 static void lock(fdb_db_t db)
@@ -49,11 +55,51 @@ static fdb_time_t get_time(void)
     return time(NULL);
 }
 
+#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT
+#include <stdlib.h>
+#include <string.h>
+static uint32_t *hash_memory;
+
+/* return the size of the memory block in byte, 
+   the memory block is composed of uint32_t
+   The block size MUST BE a multiple of four.*/
+static int hash_init(uint32_t default_value)
+{
+    size_t elements_cnt = 4096; 
+    size_t size = 4 * elements_cnt;
+
+    hash_memory = (uint32_t *)malloc(size);
+    memset(hash_memory, default_value, size);
+
+    return size;
+}
+
+/* return an element of the memory block */
+static uint32_t hash_read(long offset)
+{
+    return hash_memory[offset];
+}
+
+/* write an element to the memory block */
+static int hash_write(long offset, const uint32_t addr)
+{
+    hash_memory[offset] = addr;
+    return 1;
+}
+#endif
+
+
 int main(void)
 {
     fdb_err_t result;
     bool file_mode = true;
-    uint32_t sec_size = 4096, db_size = sec_size * 4;
+    uint32_t sec_size = 4096, db_size = sec_size * 1024;
+
+#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT
+    hash_ops.init = hash_init;
+    hash_ops.read = hash_read;
+    hash_ops.write = hash_write;
+#endif
 
 #ifdef FDB_USING_KVDB
     { /* KVDB Sample */
@@ -72,6 +118,9 @@ int main(void)
         fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_FILE_MODE, &file_mode);
         /* create database directory */
         mkdir("fdb_kvdb1", 0777);
+#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT
+        fdb_kvdb_control(&kvdb, FDB_KVDB_CTRL_SET_HASH_OPS, &hash_ops);
+#endif
         /* Key-Value database initialization
          *
          *       &kvdb: database object
@@ -93,6 +142,13 @@ int main(void)
         kvdb_type_string_sample(&kvdb);
         /* run blob KV samples */
         kvdb_type_blob_sample(&kvdb);
+
+        /* run bench test */
+        {
+            clock_t stick =  clock();
+            kvdb_bench_sample(&kvdb);
+            FDB_INFO("\nBench time: %d\n\n",clock() - stick);
+        }
     }
 #endif /* FDB_USING_KVDB */
 

docs/configuration.md

@@ -38,4 +38,7 @@ The print function macro defines the configuration. When it is not configured by
 
 ## FDB_DEBUG_ENABLE
 
-Enable debugging information output. When this configuration is closed, the system will not output logs for debugging.
+Enable debugging information output. When this configuration is closed, the system will not output logs for debugging.
+
+## FDB_KV_CACHE_HASHINDEX_ENHANCEMENT
+Enable the enhancement of the KV cache, it will take more RAM but improve the search speed, it will create a hash-base index in memory. It will take 4-bytes for a key-value entry.

inc/fdb_def.h

@@ -37,6 +37,10 @@ extern "C" {
 
 #if (FDB_KV_CACHE_TABLE_SIZE > 0) && (FDB_SECTOR_CACHE_TABLE_SIZE > 0)
 #define FDB_KV_USING_CACHE
+/* the KV cache use hashmap enhancement, it will take more RAM but improve the search speed */
+//#define FDB_KV_CACHE_HASH_ENHANCEMENT
+#else
+#undef FDB_KV_CACHE_HASH_ENHANCEMENT
 #endif
 
 #if defined(FDB_USING_FILE_LIBC_MODE) || defined(FDB_USING_FILE_POSIX_MODE)
@@ -76,6 +80,7 @@ if (!(EXPR))                                                                  \
 #define FDB_KVDB_CTRL_SET_FILE_MODE    0x09             /**< set file mode control command */
 #define FDB_KVDB_CTRL_SET_MAX_SIZE     0x0A             /**< set database max size in file mode control command */
 #define FDB_KVDB_CTRL_SET_NOT_FORMAT   0x0B             /**< set database NOT format mode control command */
+#define FDB_KVDB_CTRL_SET_HASH_OPS     0x0C             /**< set hash operations if hash enhancement is enabled */
 
 #define FDB_TSDB_CTRL_SET_SEC_SIZE     0x00             /**< set sector size control command */
 #define FDB_TSDB_CTRL_GET_SEC_SIZE     0x01             /**< get sector size control command */
@@ -281,6 +286,25 @@ struct fdb_db {
     void *user_data;
 };
 
+#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT
+/* memory ops,
+   In some complicated situations, the memory may use page-management, 
+   so the functions designed to switch pages before accessing,
+    or meet other prerequisites.
+ */
+struct fdb_cache_hash_enhancement_ops
+{
+    /* init call when need init, please return the size of memory that allocated */
+    int (*init)(uint32_t default_value);
+    /* read the offset, return a 4-bytes data */
+    uint32_t (*read)(long offset);
+    /* write a 4-bytes data */
+    int (*write)(long offset, const uint32_t addr);
+};
+
+typedef struct fdb_cache_hash_enhancement_ops *fdb_cache_hash_enhancement_ops_t;
+#endif
+
 /* KVDB structure */
 struct fdb_kvdb {
     struct fdb_db parent;                        /**< inherit from fdb_db */
@@ -296,6 +320,14 @@ struct fdb_kvdb {
     struct kv_cache_node kv_cache_table[FDB_KV_CACHE_TABLE_SIZE];
     /* sector cache table, it caching the sector info which status is current using */
     struct sector_cache_node sector_cache_table[FDB_SECTOR_CACHE_TABLE_SIZE];
+#ifdef FDB_KV_CACHE_HASH_ENHANCEMENT
+    /* KV hash enhancement */
+    bool kv_cache_enm_collisions;
+    bool kv_cache_table_collisions;
+    size_t kv_cache_enm_total_size;
+    size_t kv_cache_enm_size;
+    fdb_cache_hash_enhancement_ops_t kv_cache_enm_ops;
+#endif
 #endif /* FDB_KV_USING_CACHE */
 
 #ifdef FDB_KV_AUTO_UPDATE

samples/kvdb_benchmark_sample.c

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2020, LianYang, <[email protected]>
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+/**
+ * @file
+ * @brief benchmark KV samples.
+ *
+ * write some KV entries and read it out
+ */
+
+#include <flashdb.h>
+#include <string.h>
+
+#ifdef FDB_USING_KVDB
+
+#define FDB_LOG_TAG "[sample][kvdb][bench]"
+
+#define ROUND (200)
+
+void kvdb_bench_sample(fdb_kvdb_t kvdb)
+{
+    struct fdb_blob blob;
+    char bench_key[FDB_KV_NAME_MAX];
+    char bench_value[FDB_KV_NAME_MAX];
+
+    FDB_INFO("==================== kvdb_bench_sample ====================\n");
+
+    if (fdb_kv_get(kvdb, "1")==NULL)
+    { /* SET the KV value */
+        for (int i = 0; i<ROUND; i++) {
+            snprintf(bench_key, FDB_KV_NAME_MAX-1, "%d", i);
+            snprintf(bench_value, FDB_KV_NAME_MAX-1, "VALUE%d", i);
+
+            fdb_kv_set(kvdb, bench_key, bench_value);
+            FDB_INFO("set the '%8s' value to %12s\n", bench_key, bench_value);
+        }
+    }
+
+    { /* GET the KV value */
+        for (int i = 0; i<ROUND; i++ ) {
+            snprintf(bench_key, FDB_KV_NAME_MAX-1, "%d", i);
+
+            char* read_value = fdb_kv_get(kvdb, bench_key);
+            FDB_INFO("get the '%8s' value is %12s\n", bench_key, read_value);
+        }
+    }
+
+    FDB_INFO("===========================================================\n");
+}
+
+#endif /* FDB_USING_KVDB */