Merge branch 'develop'

Author: mkalinin

.travis.yml

@@ -4,11 +4,14 @@ dist: trusty
 language: java
 jdk: oraclejdk8
 
+before_script:
+  - cd $HOME && mkdir tests  && cd tests && git clone https://github.com/ethereum/tests.git . && cd $TRAVIS_BUILD_DIR
+
 # publish snapshots to https://oss.jfrog.org/libs-snapshot/org/ethereum/ethereumj-core
 # publish releases to http://jcenter.bintray.com/org/ethereum/ethereumj-core
 # publish coverage to https://coveralls.io/r/ethereum/ethereumj
 script:
-  - ./gradlew clean build publish jacocoTestReport coveralls --stacktrace --info
+  - ./gradlew clean build publish jacocoTestReport coveralls $TESTS_OPTS --stacktrace --info
 
 
 cache:
@@ -44,5 +47,6 @@ notifications:
 env:
   global:
     - JAVA_OPTS="-XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled"
+    - TESTS_OPTS="-DGitHubTests.testPath=$HOME/tests"
     - secure: "OlSe9DlT1D/b/ru3uO1m8nwevaDhH9XGmAfJ/2U69eBwRtg/aLEF9ZpULrMNTDR8XbNT6uuZsvvRby5HOKPRRkOqnWIY8He2hRpw0IYDONfRfKXIcr4WuJM3N98mQ9RYoNcV9LbHoXFQtfc7oUIp5o7WsCx5Pd/Ygyz4ZVNBc5g="
     - secure: "Y5L4DJXonAavfoUAMgM+RUTVYfyT5YkB8yBp8oUTK6RMCUrSTXB2Kpa8fvP8gvPXIXpIQgxa+bn/85wSrFAm8I9e4zXYO/1h4TPsbXrE1KB3aIXlg96wr1WRg+YyWed1VOtrCDZhO0K9l2fEG4ktysv+vtSaDxRVjtnFX+0Xymk="

README.md

@@ -42,7 +42,7 @@ git clone https://github.com/ethereum/ethereumj
 cd ethereumj
 cp ethereumj-core/src/main/resources/ethereumj.conf ethereumj-core/src/main/resources/user.conf
 vim ethereumj-core/src/main/resources/user.conf # adjust user.conf to your needs
-./gradlew clean shadowJar
+./gradlew clean fatJar
 java -jar ethereumj-core/build/libs/ethereumj-core-*-all.jar
 ```
 

ethereumj-core/build.gradle

@@ -8,6 +8,7 @@ buildscript {
     dependencies {
         classpath 'io.spring.gradle:propdeps-plugin:0.0.9.RELEASE'
         classpath 'org.jfrog.buildinfo:build-info-extractor-gradle:3.0.1'
+        classpath 'com.typesafe:config:1.2.1'
     }
 }
 
@@ -17,6 +18,7 @@ plugins {
     id 'jacoco'
     id 'com.github.kt3k.coveralls' version '2.6.3'
     id 'com.jfrog.bintray' version '1.0'
+    id 'org.ajoberstar.grgit' version '2.2.0' apply false
 }
 
 apply plugin: 'propdeps-maven'
@@ -78,15 +80,25 @@ task tckRun(type:JavaExec){
     classpath = sourceSets.test.runtimeClasspath
 }
 
-
-
+/**
+ * Runs all tests
+ * EthereumJ also runs common Ethereum clients tests
+ * See {@link #prepareGithubTests()} if you want to use local repo
+ * instead of downloading tests during test execution
+ */
 test {
 
     beforeTest { descriptor ->
         logger.lifecycle("Running test: ${descriptor}")
     }
 
-    jvmArgs '-Xss8m', '-Xmx3G'
+    jvmArgs = ["-Xss8m", "-Xmx3G"]
+
+    prepareGithubTests()
+    // If Github tests directory is provided, it's passed to test runner
+    if (System.getProperty("GitHubTests.testPath") != null) {
+        jvmArgs.add('-DGitHubTests.testPath' + '=' + System.getProperty("GitHubTests.testPath"))
+    }
 
     testLogging {
         events "failed"
@@ -165,6 +177,16 @@ javadoc {
     )
 }
 
+//create a single Jar with all dependencies
+task fatJar(type: Jar) {
+    classifier = 'all'
+    manifest {
+        attributes 'Main-Class' : mainClassName
+        attributes 'Class-Path' : configurations.compile.collect { 'lib/' + it.getName() }.join(' ')
+    }
+    from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
+    with jar
+}
 
 task sourcesJar(type: Jar, dependsOn: classes) {
     classifier = 'sources'
@@ -346,3 +368,73 @@ def gitCurBranch() {
     return process.text.trim()
 }
 
+/**
+ * This method is used to prepare local Ethereum Github tests for EthereumJ tests
+ *
+ * These tests  are designed for all Ethereum clients and
+ * maintained by Ethereum Foundation/Community.
+ * EthereumJ executes all of them but by default downloads every
+ * single file using internet connection.
+ * Alternatively you could use local repo to run tests from it.
+ *
+ * Common usage:
+ * 1) Local run:
+ *  a. Clone Ethereum tests repo (https://github.com/ethereum/tests)
+ *    to some local directory
+ *  b. Create `test-user.conf` file in test resources directory.
+ *    This file is under `.gitignore`, so it's your local file only.
+ *    You could override any EthereumJ configuration in it, but
+ *    for using local Ethereum tests repo you need following line:
+ *    GitHubTests.testPath=/some/path/with/Ethereum/tests/repo
+ *  c. Run `./gradlew test` task.
+ * 2) CI run:
+ *  a. Clone Ethereum tests repo (https://github.com/ethereum/tests)
+ *    to some local directory
+ *  b. Run `test` task with following parameter:
+ *    -DGitHubTests.testPath=/some/path/with/Ethereum/tests/repo
+ *
+ * In both cases this method will get commit hash from
+ * `src/test/resources/github-tests.prop` and checkout local repo
+ * to specific commit. So, EthereumJ tests will run common
+ * Ethereum client tests using local copy of repository.
+ *
+ * Repo with tests is changed often and EthereumJ is guaranteed
+ * to pass only tests from specific commit.
+ */
+import org.ajoberstar.grgit.Grgit
+import com.typesafe.config.Config
+import com.typesafe.config.ConfigFactory
+def prepareGithubTests() {
+    String testsPathKey = "GitHubTests.testPath"
+    String testsCommitKey = "GitHubTests.commit"
+    String testUserConf = "ethereumj-core/src/test/resources/test-user.conf"
+    String githubTestsProp = "ethereumj-core/src/test/resources/github-tests.prop"
+    def repoPath = System.getProperty(testsPathKey)
+    if (repoPath == null) {
+        try {
+            Config userTest = ConfigFactory.parseFile(new File(testUserConf))
+            repoPath = userTest.getString(testsPathKey)
+        } catch (Exception ex) {}
+    }
+    if (repoPath == null) return
+    def commit = System.getProperty(testsCommitKey)
+    if (commit == null) {
+        Properties testProp = new Properties();
+        testProp.load(new File(githubTestsProp).newDataInputStream());
+        commit = testProp.getProperty(testsCommitKey)
+    }
+    if (repoPath != null && commit != null) {
+        println "Tests: Checking out Ethereum Tests repo at " + repoPath + " to commit #" + commit
+        try {
+            def testsRepo = Grgit.open(dir: repoPath)
+            testsRepo.fetch()
+            testsRepo.checkout(branch: commit)
+        } catch (Exception ex) {
+            println "Error occurs while trying to checkout local Ethereum tests repo to specific commit"
+            println "Either remove " + testsPathKey + " property to use remote repo in tests or fix the issue"
+            println "Error: "
+            println ex
+            throw ex
+        }
+    }
+}

ethereumj-core/src/main/java/org/ethereum/Start.java

@@ -37,7 +37,7 @@ public static void main(String args[]) throws IOException, URISyntaxException {
         CLIInterface.call(args);
 
         final SystemProperties config = SystemProperties.getDefault();
-        final boolean actionBlocksLoader = !config.blocksLoader().equals("");
+        final boolean actionBlocksLoader = !config.blocksLoader().isEmpty();
         final boolean actionGenerateDag = !StringUtils.isEmpty(System.getProperty("ethash.blockNumber"));
 
         if (actionBlocksLoader || actionGenerateDag) {

ethereumj-core/src/main/java/org/ethereum/cli/CLIInterface.java

@@ -171,9 +171,9 @@ private static void printHelp() {
         System.out.println("-listen  <port>       -- port to listen on for incoming connections ");
         System.out.println("-connect <enode://pubKey@host:port>  -- address actively connect to  ");
         System.out.println("-connectOnly <enode://pubKey@host:port>  -- like 'connect', but will not attempt to connect to other peers  ");
-        System.out.println("");
+        System.out.println();
         System.out.println("e.g: cli -reset no -db db-1 -listen 20202 -connect enode://[email protected]:30300 ");
-        System.out.println("");
+        System.out.println();
 
     }
 

ethereumj-core/src/main/java/org/ethereum/config/CommonConfig.java

@@ -154,10 +154,14 @@ public AbstractCachedSource<byte[], byte[]> blockchainDbCache() {
         return ret;
     }
 
+    public DbSource<byte[]> keyValueDataSource(String name) {
+        return keyValueDataSource(name, DbSettings.DEFAULT);
+    }
+
     @Bean
     @Scope("prototype")
     @Primary
-    public DbSource<byte[]> keyValueDataSource(String name) {
+    public DbSource<byte[]> keyValueDataSource(String name, DbSettings settings) {
         String dataSource = systemProperties().getKeyValueDataSource();
         try {
             DbSource<byte[]> dbSource;
@@ -170,7 +174,7 @@ public DbSource<byte[]> keyValueDataSource(String name) {
                 dbSource = rocksDbDataSource();
             }
             dbSource.setName(name);
-            dbSource.init();
+            dbSource.init(settings);
             dbSources.add(dbSource);
             return dbSource;
         } finally {
@@ -193,7 +197,7 @@ protected RocksDbDataSource rocksDbDataSource() {
     public void fastSyncCleanUp() {
         byte[] fastsyncStageBytes = blockchainDB().get(FastSyncManager.FASTSYNC_DB_KEY_SYNC_STAGE);
         if (fastsyncStageBytes == null) return; // no uncompleted fast sync
-        if (!systemProperties().blocksLoader().equals("")) return; // blocks loader enabled
+        if (!systemProperties().blocksLoader().isEmpty()) return; // blocks loader enabled
 
         EthereumListener.SyncState syncStage = EthereumListener.SyncState.values()[fastsyncStageBytes[0]];
 
@@ -219,15 +223,26 @@ private void resetDataSource(Source source) {
 
     @Bean
     @Lazy
-    public DataSourceArray<BlockHeader> headerSource() {
-        DbSource<byte[]> dataSource = keyValueDataSource("headers");
-        BatchSourceWriter<byte[], byte[]> batchSourceWriter = new BatchSourceWriter<>(dataSource);
-        WriteCache.BytesKey<byte[]> writeCache = new WriteCache.BytesKey<>(batchSourceWriter, WriteCache.CacheType.SIMPLE);
-        writeCache.withSizeEstimators(MemSizeEstimator.ByteArrayEstimator, MemSizeEstimator.ByteArrayEstimator);
-        writeCache.setFlushSource(true);
-        ObjectDataSource<BlockHeader> objectDataSource = new ObjectDataSource<>(dataSource, Serializers.BlockHeaderSerializer, 0);
-        DataSourceArray<BlockHeader> dataSourceArray = new DataSourceArray<>(objectDataSource);
-        return dataSourceArray;
+    public DbSource<byte[]> headerSource() {
+        return keyValueDataSource("headers");
+    }
+
+    @Bean
+    @Lazy
+    public HeaderStore headerStore() {
+        DbSource<byte[]> dataSource = headerSource();
+
+        WriteCache.BytesKey<byte[]> cache = new WriteCache.BytesKey<>(
+                new BatchSourceWriter<>(dataSource), WriteCache.CacheType.SIMPLE);
+        cache.setFlushSource(true);
+        dbFlushManager().addCache(cache);
+
+        HeaderStore headerStore = new HeaderStore();
+        Source<byte[], byte[]> headers = new XorDataSource<>(cache, HashUtil.sha3("header".getBytes()));
+        Source<byte[], byte[]> index = new XorDataSource<>(cache, HashUtil.sha3("index".getBytes()));
+        headerStore.init(index, headers);
+
+        return headerStore;
     }
 
     @Bean
@@ -256,7 +271,11 @@ public ProgramPrecompile deserialize(byte[] stream) {
 
     @Bean
     public DbSource<byte[]> blockchainDB() {
-        return keyValueDataSource("blockchain");
+        DbSettings settings = DbSettings.newInstance()
+                .withMaxOpenFiles(systemProperties().getConfig().getInt("database.maxOpenFiles"))
+                .withMaxThreads(Math.max(1, Runtime.getRuntime().availableProcessors() / 2));
+
+        return keyValueDataSource("blockchain", settings);
     }
 
     @Bean

ethereumj-core/src/main/java/org/ethereum/config/NodeFilter.java

@@ -65,14 +65,19 @@ public Entry(byte[] nodeId, String hostIpPattern) {
         }
 
         public boolean accept(InetAddress nodeAddr) {
+            if (hostIpPattern == null) return true;
             String ip = nodeAddr.getHostAddress();
             return hostIpPattern != null && ip.startsWith(hostIpPattern);
         }
 
         public boolean accept(Node node) {
             try {
-                return (nodeId == null || Arrays.equals(node.getId(), nodeId))
-                        && (hostIpPattern == null || accept(InetAddress.getByName(node.getHost())));
+                boolean shouldAcceptNodeId = nodeId == null || Arrays.equals(node.getId(), nodeId);
+                if (!shouldAcceptNodeId) {
+                    return false;
+                }
+                InetAddress nodeAddress = InetAddress.getByName(node.getHost());
+                return (hostIpPattern == null || accept(nodeAddress));
             } catch (UnknownHostException e) {
                 return false;
             }

ethereumj-core/src/main/java/org/ethereum/config/SystemProperties.java

@@ -34,6 +34,7 @@
 import org.ethereum.net.rlpx.Node;
 import org.ethereum.util.BuildInfo;
 import org.ethereum.util.ByteUtil;
+import org.ethereum.util.Utils;
 import org.ethereum.validator.BlockCustomHashRule;
 import org.ethereum.validator.BlockHeaderValidator;
 import org.slf4j.Logger;
@@ -53,6 +54,7 @@
 import java.util.*;
 
 import static org.ethereum.crypto.HashUtil.sha3;
+import static org.ethereum.util.ByteUtil.toHexString;
 
 /**
  * Utility class to retrieve property values from the ethereumj.conf files
@@ -664,7 +666,7 @@ public String customSolcPath() {
     public String privateKey() {
         if (config.hasPath("peer.privateKey")) {
             String key = config.getString("peer.privateKey");
-            if (key.length() != 64) {
+            if (key.length() != 64 || !Utils.isHexEncoded(key)) {
                 throw new RuntimeException("The peer.privateKey needs to be Hex encoded and 32 byte length");
             }
             return key;
@@ -810,7 +812,7 @@ public boolean isFastSyncEnabled() {
     public byte[] getFastSyncPivotBlockHash() {
         if (!config.hasPath("sync.fast.pivotBlockHash")) return null;
         byte[] ret = Hex.decode(config.getString("sync.fast.pivotBlockHash"));
-        if (ret.length != 32) throw new RuntimeException("Invalid block hash length: " + Hex.toHexString(ret));
+        if (ret.length != 32) throw new RuntimeException("Invalid block hash length: " + toHexString(ret));
         return ret;
     }
 
@@ -824,6 +826,12 @@ public boolean fastSyncSkipHistory() {
         return config.getBoolean("sync.fast.skipHistory");
     }
 
+    @ValidateMe
+    public int makeDoneByTimeout() {
+        return config.getInt("sync.makeDoneByTimeout");
+    }
+
+
     @ValidateMe
     public boolean isPublicHomeNode() { return config.getBoolean("peer.discovery.public.home.node");}
 

ethereumj-core/src/main/java/org/ethereum/config/blockchain/ByzantiumConfig.java

@@ -23,7 +23,6 @@
 import org.ethereum.core.Block;
 import org.ethereum.core.BlockHeader;
 import org.ethereum.core.Repository;
-import org.spongycastle.util.encoders.Hex;
 
 import java.math.BigInteger;
 

ethereumj-core/src/main/java/org/ethereum/core/AccountState.java

@@ -25,12 +25,11 @@
 import org.ethereum.util.RLP;
 import org.ethereum.util.RLPList;
 
-import org.spongycastle.util.encoders.Hex;
-
 import java.math.BigInteger;
 
 import static org.ethereum.crypto.HashUtil.*;
 import static org.ethereum.util.FastByteComparisons.equal;
+import static org.ethereum.util.ByteUtil.toHexString;
 
 public class AccountState {
 
@@ -151,8 +150,8 @@ public boolean isEmpty() {
     public String toString() {
         String ret = "  Nonce: " + this.getNonce().toString() + "\n" +
                 "  Balance: " + getBalance() + "\n" +
-                "  State Root: " + Hex.toHexString(this.getStateRoot()) + "\n" +
-                "  Code Hash: " + Hex.toHexString(this.getCodeHash());
+                "  State Root: " + toHexString(this.getStateRoot()) + "\n" +
+                "  Code Hash: " + toHexString(this.getCodeHash());
         return ret;
     }
 }

ethereumj-core/src/main/java/org/ethereum/core/Block.java

@@ -34,6 +34,7 @@
 
 import static org.ethereum.crypto.HashUtil.sha3;
 import static org.ethereum.datasource.MemSizeEstimator.ByteArrayEstimator;
+import static org.ethereum.util.ByteUtil.toHexString;
 
 /**
  * The block in Ethereum is the collection of relevant pieces of information
@@ -69,7 +70,7 @@ private Block() {
     }
 
     public Block(byte[] rawData) {
-        logger.debug("new from [" + Hex.toHexString(rawData) + "]");
+        logger.debug("new from [" + toHexString(rawData) + "]");
         this.rlpEncoded = rawData;
     }
 
@@ -304,7 +305,7 @@ public String toString() {
         parseRLP();
 
         toStringBuff.setLength(0);
-        toStringBuff.append(Hex.toHexString(this.getEncoded())).append("\n");
+        toStringBuff.append(toHexString(this.getEncoded())).append("\n");
         toStringBuff.append("BlockData [ ");
         toStringBuff.append(header.toString());