[mycpp/runtime] Fix hash table pileup perf bug for mops::BigInt

I just added this hash function.

But I also fixed the one for int (32-bit).

Author: Andy C

demo/sparse-array.sh

@@ -26,6 +26,41 @@ set -o nounset
 set -o pipefail
 set -o errexit
 
+compare-x() {
+  local x=$1
+
+  local osh=_bin/cxx-opt/osh
+  ninja $osh
+
+  echo ===
+  echo $osh SparseArray
+  echo
+  time sparse-$x $osh
+
+  for sh in bash $osh; do
+    echo ===
+    echo $sh
+    echo
+    time $sh $0 $x
+  done
+}
+
+compare-sum-shift() {
+  compare-x sum-shift
+}
+
+compare-append-sparse() {
+  compare-x append-sparse
+}
+
+compare-append-dense() {
+  compare-x append-dense
+}
+
+#
+# Workloads
+#
+
 sum-shift() {
   local n=${1:-1000}
 
@@ -112,26 +147,6 @@ f
 EOF
 }
 
-compare-sum-shift() {
-  # more like 1M iterations - 1.8 seconds in bash
-  # So that's 1.8 ms for 1000 iterations
-
-  local osh=_bin/cxx-opt/osh
-  ninja $osh
-
-  echo ===
-  echo $osh SparseArray
-  echo
-  time sparse-sum-shift $osh
-
-  for sh in bash $osh; do
-    echo ===
-    echo $sh
-    echo
-    time $sh $0 sum-shift
-  done
-}
-
 append-sparse() {
   local n=${1:-24}  # up to 2^n
   local m=${2:-2000}
@@ -182,21 +197,52 @@ f
 EOF
 }
 
-compare-append-sparse() {
-  local osh=_bin/cxx-opt/osh
-  ninja $osh
+append-dense() {
+  local n=${1:-24}  # up to 2^n
+  local m=${2:-2000}
 
-  echo ===
-  echo $osh SparseArray
-  echo
-  time sparse-append-sparse $osh
+  to_append=( $(seq $m) )  # split words
 
-  for sh in bash $osh; do
-    echo ===
-    echo $sh
-    echo
-    time $sh $0 append-sparse
-  done
+  a=()
+  for (( i = 0; i < n; ++i )) {
+    a+=( ${to_append[@]} )
+  }
+  #echo ${a[@]}
+  #echo ${!a[@]}
+  echo ${#a[@]}
+}
+
+sparse-append-dense() {
+  local osh=${1:-_bin/cxx-opt/osh}
+  local n=${2:-24}
+  local m=${3:-2000}
+
+  NUM_ITERS=$n TO_APPEND=$m $osh <<'EOF'
+n=$NUM_ITERS
+m=$TO_APPEND
+to_append=( $(seq $m) )  # split words before ysh:upgrade
+
+shopt --set ysh:upgrade
+
+f() {
+  a=()
+  var sp = _a2sp(a)
+
+  for (( i = 0; i < n; ++i )) {
+    #call _opsp(sp, 'set', 1 << i, str(i))
+    call _opsp(sp, 'append', to_append)
+
+    #time call _opsp(sp, 'append', to_append)
+    #echo $[_opsp(sp, 'len')]
+    #echo
+  }
+  echo $[_opsp(sp, 'len')]
+  #echo @[_opsp(sp, 'subst')]
+  #echo @[_opsp(sp, 'keys')]
+}
+
+f
+EOF
 }
 
 demo() {

mycpp/gc_dict.h

@@ -233,6 +233,7 @@ void Dict<K, V>::reserve(int num_desired) {
   values_ = NewSlab<V>(capacity_);
 
   if (old_k != nullptr) {  // rehash if there were any entries
+    //log("REHASH num_desired %d", num_desired);
     len_ = 0;
     for (int i = 0; i < old_len; ++i) {
       set(old_k->items_[i], old_v->items_[i]);

mycpp/gc_dict_test.cc

@@ -1,5 +1,7 @@
 #include "mycpp/gc_dict.h"
 
+#include <unordered_map>
+
 #include "mycpp/gc_mylib.h"
 #include "vendor/greatest.h"
 
@@ -686,6 +688,31 @@ TEST test_hash() {
   PASS();
 }
 
+TEST hash_pileup_bug() {
+  auto* d = Alloc<Dict<mops::BigInt, BigStr*>>();
+
+  std::unordered_map<unsigned, bool> hist;
+
+  for (int i = 0; i < 24; ++i) {
+    mops::BigInt index {1 << i};
+    log("index %ld", index);
+
+    for (mops::BigInt j = index; j < index + 2000; ++j) {
+      d->set(j, kEmptyString);
+      unsigned h = hash_key(j);
+      hist[h] = true;
+      // log("%ld %d", j, h);
+    }
+    log("len %d", len(d));
+  }
+
+  log("len %d", len(d));
+  log("unique hashes %d", hist.size());
+
+  PASS();
+}
+
+
 GREATEST_MAIN_DEFS();
 
 int main(int argc, char** argv) {
@@ -710,6 +737,7 @@ int main(int argc, char** argv) {
   RUN_TEST(dict_iters_test);
 
   RUN_TEST(test_hash);
+  RUN_TEST(hash_pileup_bug);
 
   gHeap.CleanProcessExit();
 

mycpp/hash.cc

@@ -21,12 +21,19 @@ unsigned hash_key(BigStr* s) {
 }
 
 unsigned hash_key(int n) {
-  return n;
+  return fnv1(reinterpret_cast<const char*>(&n), sizeof(n));
 }
 
 unsigned hash_key(mops::BigInt n) {
-  // Cast to a smaller type.  Is there anything better we can do?
-  return static_cast<unsigned>(n);
+  // Bug fix: our dict sizing is a power of 2, and we don't want integers in
+  // the workload to interact badly with it.
+  return fnv1(reinterpret_cast<const char*>(&n), sizeof(n));
+}
+
+unsigned hash_key(void* p) {
+  // e.g. for Dict<Token*, int>, hash the pointer itself, which means we use
+  // object IDENTITY, not value.
+  return fnv1(reinterpret_cast<const char*>(&p), sizeof(void*));
 }
 
 unsigned hash_key(Tuple2<int, int>* t1) {
@@ -36,9 +43,3 @@ unsigned hash_key(Tuple2<int, int>* t1) {
 unsigned hash_key(Tuple2<BigStr*, int>* t1) {
   return t1->at0()->hash(fnv1) + t1->at1();
 }
-
-// e.g. for Dict<Token*, int>, hash the pointer itself, which means we use
-// object IDENTITY, not value.
-unsigned hash_key(void* p) {
-  return fnv1(reinterpret_cast<const char*>(&p), sizeof(void*));
-}