Merge pull request #1061 from ethereum/fix/slow-import

Fix slow import

Author: zilm13

ethereumj-core/src/main/java/org/ethereum/config/CommonConfig.java

@@ -154,10 +154,14 @@ public AbstractCachedSource<byte[], byte[]> blockchainDbCache() {
         return ret;
     }
 
+    public DbSource<byte[]> keyValueDataSource(String name) {
+        return keyValueDataSource(name, DbSettings.DEFAULT);
+    }
+
     @Bean
     @Scope("prototype")
     @Primary
-    public DbSource<byte[]> keyValueDataSource(String name) {
+    public DbSource<byte[]> keyValueDataSource(String name, DbSettings settings) {
         String dataSource = systemProperties().getKeyValueDataSource();
         try {
             DbSource<byte[]> dbSource;
@@ -170,7 +174,7 @@ public DbSource<byte[]> keyValueDataSource(String name) {
                 dbSource = rocksDbDataSource();
             }
             dbSource.setName(name);
-            dbSource.init();
+            dbSource.init(settings);
             dbSources.add(dbSource);
             return dbSource;
         } finally {
@@ -267,7 +271,11 @@ public ProgramPrecompile deserialize(byte[] stream) {
 
     @Bean
     public DbSource<byte[]> blockchainDB() {
-        return keyValueDataSource("blockchain");
+        DbSettings settings = DbSettings.newInstance()
+                .withMaxOpenFiles(systemProperties().getConfig().getInt("database.maxOpenFiles"))
+                .withMaxThreads(Math.max(1, Runtime.getRuntime().availableProcessors() / 2));
+
+        return keyValueDataSource("blockchain", settings);
     }
 
     @Bean

ethereumj-core/src/main/java/org/ethereum/datasource/DbSettings.java

@@ -0,0 +1,45 @@
+package org.ethereum.datasource;
+
+/**
+ * Defines configurable database settings
+ *
+ * @author Mikhail Kalinin
+ * @since 26.04.2018
+ */
+public class DbSettings {
+
+    public static final DbSettings DEFAULT = new DbSettings()
+            .withMaxThreads(1)
+            .withMaxOpenFiles(32);
+
+    int maxOpenFiles;
+    int maxThreads;
+
+    private DbSettings() {
+    }
+
+    public static DbSettings newInstance() {
+        DbSettings settings = new DbSettings();
+        settings.maxOpenFiles = DEFAULT.maxOpenFiles;
+        settings.maxThreads = DEFAULT.maxThreads;
+        return settings;
+    }
+
+    public int getMaxOpenFiles() {
+        return maxOpenFiles;
+    }
+
+    public DbSettings withMaxOpenFiles(int maxOpenFiles) {
+        this.maxOpenFiles = maxOpenFiles;
+        return this;
+    }
+
+    public int getMaxThreads() {
+        return maxThreads;
+    }
+
+    public DbSettings withMaxThreads(int maxThreads) {
+        this.maxThreads = maxThreads;
+        return this;
+    }
+}

ethereumj-core/src/main/java/org/ethereum/datasource/DbSource.java

@@ -38,7 +38,7 @@ public interface DbSource<V> extends BatchSource<byte[], V> {
     /**
      * Initializes DB (open table, connection, etc)
      */
-    void init();
+    void init(DbSettings settings);
 
     /**
      * @return true if DB connection is alive

ethereumj-core/src/main/java/org/ethereum/datasource/inmem/HashMapDB.java

@@ -17,14 +17,14 @@
  */
 package org.ethereum.datasource.inmem;
 
+import org.ethereum.datasource.DbSettings;
 import org.ethereum.datasource.DbSource;
 import org.ethereum.util.ALock;
 import org.ethereum.util.ByteArrayMap;
 import org.ethereum.util.FastByteComparisons;
 
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.locks.Lock;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 
@@ -86,7 +86,7 @@ public String getName() {
     }
 
     @Override
-    public void init() {}
+    public void init(DbSettings settings) {}
 
     @Override
     public boolean isAlive() {

ethereumj-core/src/main/java/org/ethereum/datasource/inmem/HashMapDBSimple.java

@@ -17,15 +17,13 @@
  */
 package org.ethereum.datasource.inmem;
 
+import org.ethereum.datasource.DbSettings;
 import org.ethereum.datasource.DbSource;
-import org.ethereum.util.ALock;
 import org.ethereum.util.ByteArrayMap;
 import org.ethereum.util.FastByteComparisons;
 
 import java.util.Map;
 import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
 
 /**
  * Created by Anton Nashatyrev on 12.10.2016.
@@ -75,7 +73,7 @@ public String getName() {
     }
 
     @Override
-    public void init() {}
+    public void init(DbSettings settings) {}
 
     @Override
     public boolean isAlive() {

ethereumj-core/src/main/java/org/ethereum/datasource/leveldb/LevelDbDataSource.java

@@ -18,6 +18,7 @@
 package org.ethereum.datasource.leveldb;
 
 import org.ethereum.config.SystemProperties;
+import org.ethereum.datasource.DbSettings;
 import org.ethereum.datasource.DbSource;
 import org.ethereum.util.FileUtil;
 import org.iq80.leveldb.*;
@@ -54,6 +55,8 @@ public class LevelDbDataSource implements DbSource<byte[]> {
     DB db;
     boolean alive;
 
+    DbSettings settings = DbSettings.DEFAULT;
+
     // The native LevelDB insert/update/delete are normally thread-safe
     // However close operation is not thread-safe and may lead to a native crash when
     // accessing a closed DB.
@@ -71,7 +74,8 @@ public LevelDbDataSource(String name) {
     }
 
     @Override
-    public void init() {
+    public void init(DbSettings settings) {
+        this.settings = settings;
         resetDbLock.writeLock().lock();
         try {
             logger.debug("~> LevelDbDataSource.init(): " + name);
@@ -88,7 +92,7 @@ public void init() {
             options.cacheSize(0);
             options.paranoidChecks(true);
             options.verifyChecksums(true);
-            options.maxOpenFiles(32);
+            options.maxOpenFiles(settings.getMaxOpenFiles());
 
             try {
                 logger.debug("Opening database");
@@ -135,7 +139,7 @@ private Path getPath() {
     public void reset() {
         close();
         FileUtil.recursiveDelete(getPath().toString());
-        init();
+        init(settings);
     }
 
     @Override

ethereumj-core/src/main/java/org/ethereum/datasource/rocksdb/RocksDbDataSource.java

@@ -18,6 +18,7 @@
 package org.ethereum.datasource.rocksdb;
 
 import org.ethereum.config.SystemProperties;
+import org.ethereum.datasource.DbSettings;
 import org.ethereum.datasource.DbSource;
 import org.ethereum.datasource.NodeKeyCompositor;
 import org.ethereum.util.FileUtil;
@@ -56,6 +57,8 @@ public class RocksDbDataSource implements DbSource<byte[]> {
     ReadOptions readOpts;
     boolean alive;
 
+    DbSettings settings = DbSettings.DEFAULT;
+
     // The native RocksDB insert/update/delete are normally thread-safe
     // However close operation is not thread-safe.
     // This ReadWriteLock still permits concurrent execution of insert/delete/update operations
@@ -85,7 +88,8 @@ public String getName() {
     }
 
     @Override
-    public void init() {
+    public void init(DbSettings settings) {
+        this.settings = settings;
         resetDbLock.writeLock().lock();
         try {
             logger.debug("~> RocksDbDataSource.init(): " + name);
@@ -103,9 +107,8 @@ public void init() {
                 options.setCompressionType(CompressionType.LZ4_COMPRESSION);
                 options.setBottommostCompressionType(CompressionType.ZSTD_COMPRESSION);
                 options.setLevelCompactionDynamicLevelBytes(true);
-                options.setMaxBackgroundCompactions(4);
-                options.setMaxBackgroundFlushes(2);
-                options.setMaxOpenFiles(32);
+                options.setMaxOpenFiles(settings.getMaxOpenFiles());
+                options.setIncreaseParallelism(settings.getMaxThreads());
 
                 // key prefix for state node lookups
                 options.useFixedLengthPrefixExtractor(NodeKeyCompositor.PREFIX_BYTES);
@@ -237,7 +240,7 @@ public Set<byte[]> keys() throws RuntimeException {
     public void reset() {
         close();
         FileUtil.recursiveDelete(getPath().toString());
-        init();
+        init(settings);
     }
 
     private Path getPath() {

ethereumj-core/src/main/resources/ethereumj.conf

@@ -218,6 +218,12 @@ database {
         # 1_000_000: 5658 Mb
         maxDepth = 192
     }
+
+    # defines a number of opened files by db instance
+    # this number has significant impact on read amplification
+    # on the other hand it can force exceeding of user's limit,
+    # OS usually set it to 1024 for all applications
+    maxOpenFiles = 512
 }
 
 # Cache settings
@@ -239,7 +245,7 @@ cache {
     }
 
     # total size in Mbytes of the state DB read cache
-    stateCacheSize = 256
+    stateCacheSize = 384
 
     # the size of block queue cache to be imported in MBytes
     blockQueueSize = 32