Skip to content

Commit

Permalink
Merge branch 'develop'
Browse files Browse the repository at this point in the history
  • Loading branch information
mkalinin committed Jun 14, 2018
2 parents b17cb8a + c887e11 commit 4430b41
Show file tree
Hide file tree
Showing 149 changed files with 2,773 additions and 775 deletions.
6 changes: 5 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@ dist: trusty
language: java
jdk: oraclejdk8

before_script:
- cd $HOME && mkdir tests && cd tests && git clone https://github.com/ethereum/tests.git . && cd $TRAVIS_BUILD_DIR

# publish snapshots to https://oss.jfrog.org/libs-snapshot/org/ethereum/ethereumj-core
# publish releases to http://jcenter.bintray.com/org/ethereum/ethereumj-core
# publish coverage to https://coveralls.io/r/ethereum/ethereumj
script:
- ./gradlew clean build publish jacocoTestReport coveralls --stacktrace --info
- ./gradlew clean build publish jacocoTestReport coveralls $TESTS_OPTS --stacktrace --info


cache:
Expand Down Expand Up @@ -44,5 +47,6 @@ notifications:
env:
global:
- JAVA_OPTS="-XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled"
- TESTS_OPTS="-DGitHubTests.testPath=$HOME/tests"
- secure: "OlSe9DlT1D/b/ru3uO1m8nwevaDhH9XGmAfJ/2U69eBwRtg/aLEF9ZpULrMNTDR8XbNT6uuZsvvRby5HOKPRRkOqnWIY8He2hRpw0IYDONfRfKXIcr4WuJM3N98mQ9RYoNcV9LbHoXFQtfc7oUIp5o7WsCx5Pd/Ygyz4ZVNBc5g="
- secure: "Y5L4DJXonAavfoUAMgM+RUTVYfyT5YkB8yBp8oUTK6RMCUrSTXB2Kpa8fvP8gvPXIXpIQgxa+bn/85wSrFAm8I9e4zXYO/1h4TPsbXrE1KB3aIXlg96wr1WRg+YyWed1VOtrCDZhO0K9l2fEG4ktysv+vtSaDxRVjtnFX+0Xymk="
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ git clone https://github.com/ethereum/ethereumj
cd ethereumj
cp ethereumj-core/src/main/resources/ethereumj.conf ethereumj-core/src/main/resources/user.conf
vim ethereumj-core/src/main/resources/user.conf # adjust user.conf to your needs
./gradlew clean shadowJar
./gradlew clean fatJar
java -jar ethereumj-core/build/libs/ethereumj-core-*-all.jar
```

Expand Down
98 changes: 95 additions & 3 deletions ethereumj-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ buildscript {
dependencies {
classpath 'io.spring.gradle:propdeps-plugin:0.0.9.RELEASE'
classpath 'org.jfrog.buildinfo:build-info-extractor-gradle:3.0.1'
classpath 'com.typesafe:config:1.2.1'
}
}

Expand All @@ -17,6 +18,7 @@ plugins {
id 'jacoco'
id 'com.github.kt3k.coveralls' version '2.6.3'
id 'com.jfrog.bintray' version '1.0'
id 'org.ajoberstar.grgit' version '2.2.0' apply false
}

apply plugin: 'propdeps-maven'
Expand Down Expand Up @@ -78,15 +80,25 @@ task tckRun(type:JavaExec){
classpath = sourceSets.test.runtimeClasspath
}



/**
* Runs all tests
* EthereumJ also runs common Ethereum clients tests
* See {@link #prepareGithubTests()} if you want to use local repo
* instead of downloading tests during test execution
*/
test {

beforeTest { descriptor ->
logger.lifecycle("Running test: ${descriptor}")
}

jvmArgs '-Xss8m', '-Xmx3G'
jvmArgs = ["-Xss8m", "-Xmx3G"]

prepareGithubTests()
// If Github tests directory is provided, it's passed to test runner
if (System.getProperty("GitHubTests.testPath") != null) {
jvmArgs.add('-DGitHubTests.testPath' + '=' + System.getProperty("GitHubTests.testPath"))
}

testLogging {
events "failed"
Expand Down Expand Up @@ -165,6 +177,16 @@ javadoc {
)
}

//create a single Jar with all dependencies
task fatJar(type: Jar) {
classifier = 'all'
manifest {
attributes 'Main-Class' : mainClassName
attributes 'Class-Path' : configurations.compile.collect { 'lib/' + it.getName() }.join(' ')
}
from { configurations.compile.collect { it.isDirectory() ? it : zipTree(it) } }
with jar
}

task sourcesJar(type: Jar, dependsOn: classes) {
classifier = 'sources'
Expand Down Expand Up @@ -346,3 +368,73 @@ def gitCurBranch() {
return process.text.trim()
}

/**
* This method is used to prepare local Ethereum Github tests for EthereumJ tests
*
* These tests are designed for all Ethereum clients and
* maintained by Ethereum Foundation/Community.
* EthereumJ executes all of them but by default downloads every
* single file using internet connection.
* Alternatively you could use local repo to run tests from it.
*
* Common usage:
* 1) Local run:
* a. Clone Ethereum tests repo (https://github.com/ethereum/tests)
* to some local directory
* b. Create `test-user.conf` file in test resources directory.
* This file is under `.gitignore`, so it's your local file only.
* You could override any EthereumJ configuration in it, but
* for using local Ethereum tests repo you need following line:
* GitHubTests.testPath=/some/path/with/Ethereum/tests/repo
* c. Run `./gradlew test` task.
* 2) CI run:
* a. Clone Ethereum tests repo (https://github.com/ethereum/tests)
* to some local directory
* b. Run `test` task with following parameter:
* -DGitHubTests.testPath=/some/path/with/Ethereum/tests/repo
*
* In both cases this method will get commit hash from
* `src/test/resources/github-tests.prop` and checkout local repo
* to specific commit. So, EthereumJ tests will run common
* Ethereum client tests using local copy of repository.
*
* Repo with tests is changed often and EthereumJ is guaranteed
* to pass only tests from specific commit.
*/
import org.ajoberstar.grgit.Grgit
import com.typesafe.config.Config
import com.typesafe.config.ConfigFactory
def prepareGithubTests() {
String testsPathKey = "GitHubTests.testPath"
String testsCommitKey = "GitHubTests.commit"
String testUserConf = "ethereumj-core/src/test/resources/test-user.conf"
String githubTestsProp = "ethereumj-core/src/test/resources/github-tests.prop"
def repoPath = System.getProperty(testsPathKey)
if (repoPath == null) {
try {
Config userTest = ConfigFactory.parseFile(new File(testUserConf))
repoPath = userTest.getString(testsPathKey)
} catch (Exception ex) {}
}
if (repoPath == null) return
def commit = System.getProperty(testsCommitKey)
if (commit == null) {
Properties testProp = new Properties();
testProp.load(new File(githubTestsProp).newDataInputStream());
commit = testProp.getProperty(testsCommitKey)
}
if (repoPath != null && commit != null) {
println "Tests: Checking out Ethereum Tests repo at " + repoPath + " to commit #" + commit
try {
def testsRepo = Grgit.open(dir: repoPath)
testsRepo.fetch()
testsRepo.checkout(branch: commit)
} catch (Exception ex) {
println "Error occurs while trying to checkout local Ethereum tests repo to specific commit"
println "Either remove " + testsPathKey + " property to use remote repo in tests or fix the issue"
println "Error: "
println ex
throw ex
}
}
}
2 changes: 1 addition & 1 deletion ethereumj-core/src/main/java/org/ethereum/Start.java
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public static void main(String args[]) throws IOException, URISyntaxException {
CLIInterface.call(args);

final SystemProperties config = SystemProperties.getDefault();
final boolean actionBlocksLoader = !config.blocksLoader().equals("");
final boolean actionBlocksLoader = !config.blocksLoader().isEmpty();
final boolean actionGenerateDag = !StringUtils.isEmpty(System.getProperty("ethash.blockNumber"));

if (actionBlocksLoader || actionGenerateDag) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,9 +171,9 @@ private static void printHelp() {
System.out.println("-listen <port> -- port to listen on for incoming connections ");
System.out.println("-connect <enode://pubKey@host:port> -- address actively connect to ");
System.out.println("-connectOnly <enode://pubKey@host:port> -- like 'connect', but will not attempt to connect to other peers ");
System.out.println("");
System.out.println();
System.out.println("e.g: cli -reset no -db db-1 -listen 20202 -connect enode://[email protected]:30300 ");
System.out.println("");
System.out.println();

}

Expand Down
45 changes: 32 additions & 13 deletions ethereumj-core/src/main/java/org/ethereum/config/CommonConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -154,10 +154,14 @@ public AbstractCachedSource<byte[], byte[]> blockchainDbCache() {
return ret;
}

public DbSource<byte[]> keyValueDataSource(String name) {
return keyValueDataSource(name, DbSettings.DEFAULT);
}

@Bean
@Scope("prototype")
@Primary
public DbSource<byte[]> keyValueDataSource(String name) {
public DbSource<byte[]> keyValueDataSource(String name, DbSettings settings) {
String dataSource = systemProperties().getKeyValueDataSource();
try {
DbSource<byte[]> dbSource;
Expand All @@ -170,7 +174,7 @@ public DbSource<byte[]> keyValueDataSource(String name) {
dbSource = rocksDbDataSource();
}
dbSource.setName(name);
dbSource.init();
dbSource.init(settings);
dbSources.add(dbSource);
return dbSource;
} finally {
Expand All @@ -193,7 +197,7 @@ protected RocksDbDataSource rocksDbDataSource() {
public void fastSyncCleanUp() {
byte[] fastsyncStageBytes = blockchainDB().get(FastSyncManager.FASTSYNC_DB_KEY_SYNC_STAGE);
if (fastsyncStageBytes == null) return; // no uncompleted fast sync
if (!systemProperties().blocksLoader().equals("")) return; // blocks loader enabled
if (!systemProperties().blocksLoader().isEmpty()) return; // blocks loader enabled

EthereumListener.SyncState syncStage = EthereumListener.SyncState.values()[fastsyncStageBytes[0]];

Expand All @@ -219,15 +223,26 @@ private void resetDataSource(Source source) {

@Bean
@Lazy
public DataSourceArray<BlockHeader> headerSource() {
DbSource<byte[]> dataSource = keyValueDataSource("headers");
BatchSourceWriter<byte[], byte[]> batchSourceWriter = new BatchSourceWriter<>(dataSource);
WriteCache.BytesKey<byte[]> writeCache = new WriteCache.BytesKey<>(batchSourceWriter, WriteCache.CacheType.SIMPLE);
writeCache.withSizeEstimators(MemSizeEstimator.ByteArrayEstimator, MemSizeEstimator.ByteArrayEstimator);
writeCache.setFlushSource(true);
ObjectDataSource<BlockHeader> objectDataSource = new ObjectDataSource<>(dataSource, Serializers.BlockHeaderSerializer, 0);
DataSourceArray<BlockHeader> dataSourceArray = new DataSourceArray<>(objectDataSource);
return dataSourceArray;
public DbSource<byte[]> headerSource() {
return keyValueDataSource("headers");
}

@Bean
@Lazy
public HeaderStore headerStore() {
DbSource<byte[]> dataSource = headerSource();

WriteCache.BytesKey<byte[]> cache = new WriteCache.BytesKey<>(
new BatchSourceWriter<>(dataSource), WriteCache.CacheType.SIMPLE);
cache.setFlushSource(true);
dbFlushManager().addCache(cache);

HeaderStore headerStore = new HeaderStore();
Source<byte[], byte[]> headers = new XorDataSource<>(cache, HashUtil.sha3("header".getBytes()));
Source<byte[], byte[]> index = new XorDataSource<>(cache, HashUtil.sha3("index".getBytes()));
headerStore.init(index, headers);

return headerStore;
}

@Bean
Expand Down Expand Up @@ -256,7 +271,11 @@ public ProgramPrecompile deserialize(byte[] stream) {

@Bean
public DbSource<byte[]> blockchainDB() {
return keyValueDataSource("blockchain");
DbSettings settings = DbSettings.newInstance()
.withMaxOpenFiles(systemProperties().getConfig().getInt("database.maxOpenFiles"))
.withMaxThreads(Math.max(1, Runtime.getRuntime().availableProcessors() / 2));

return keyValueDataSource("blockchain", settings);
}

@Bean
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,14 +65,19 @@ public Entry(byte[] nodeId, String hostIpPattern) {
}

public boolean accept(InetAddress nodeAddr) {
if (hostIpPattern == null) return true;
String ip = nodeAddr.getHostAddress();
return hostIpPattern != null && ip.startsWith(hostIpPattern);
}

public boolean accept(Node node) {
try {
return (nodeId == null || Arrays.equals(node.getId(), nodeId))
&& (hostIpPattern == null || accept(InetAddress.getByName(node.getHost())));
boolean shouldAcceptNodeId = nodeId == null || Arrays.equals(node.getId(), nodeId);
if (!shouldAcceptNodeId) {
return false;
}
InetAddress nodeAddress = InetAddress.getByName(node.getHost());
return (hostIpPattern == null || accept(nodeAddress));
} catch (UnknownHostException e) {
return false;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.ethereum.net.rlpx.Node;
import org.ethereum.util.BuildInfo;
import org.ethereum.util.ByteUtil;
import org.ethereum.util.Utils;
import org.ethereum.validator.BlockCustomHashRule;
import org.ethereum.validator.BlockHeaderValidator;
import org.slf4j.Logger;
Expand All @@ -53,6 +54,7 @@
import java.util.*;

import static org.ethereum.crypto.HashUtil.sha3;
import static org.ethereum.util.ByteUtil.toHexString;

/**
* Utility class to retrieve property values from the ethereumj.conf files
Expand Down Expand Up @@ -664,7 +666,7 @@ public String customSolcPath() {
public String privateKey() {
if (config.hasPath("peer.privateKey")) {
String key = config.getString("peer.privateKey");
if (key.length() != 64) {
if (key.length() != 64 || !Utils.isHexEncoded(key)) {
throw new RuntimeException("The peer.privateKey needs to be Hex encoded and 32 byte length");
}
return key;
Expand Down Expand Up @@ -810,7 +812,7 @@ public boolean isFastSyncEnabled() {
public byte[] getFastSyncPivotBlockHash() {
if (!config.hasPath("sync.fast.pivotBlockHash")) return null;
byte[] ret = Hex.decode(config.getString("sync.fast.pivotBlockHash"));
if (ret.length != 32) throw new RuntimeException("Invalid block hash length: " + Hex.toHexString(ret));
if (ret.length != 32) throw new RuntimeException("Invalid block hash length: " + toHexString(ret));
return ret;
}

Expand All @@ -824,6 +826,12 @@ public boolean fastSyncSkipHistory() {
return config.getBoolean("sync.fast.skipHistory");
}

@ValidateMe
public int makeDoneByTimeout() {
return config.getInt("sync.makeDoneByTimeout");
}


@ValidateMe
public boolean isPublicHomeNode() { return config.getBoolean("peer.discovery.public.home.node");}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
import org.ethereum.core.Block;
import org.ethereum.core.BlockHeader;
import org.ethereum.core.Repository;
import org.spongycastle.util.encoders.Hex;

import java.math.BigInteger;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,11 @@
import org.ethereum.util.RLP;
import org.ethereum.util.RLPList;

import org.spongycastle.util.encoders.Hex;

import java.math.BigInteger;

import static org.ethereum.crypto.HashUtil.*;
import static org.ethereum.util.FastByteComparisons.equal;
import static org.ethereum.util.ByteUtil.toHexString;

public class AccountState {

Expand Down Expand Up @@ -151,8 +150,8 @@ public boolean isEmpty() {
public String toString() {
String ret = " Nonce: " + this.getNonce().toString() + "\n" +
" Balance: " + getBalance() + "\n" +
" State Root: " + Hex.toHexString(this.getStateRoot()) + "\n" +
" Code Hash: " + Hex.toHexString(this.getCodeHash());
" State Root: " + toHexString(this.getStateRoot()) + "\n" +
" Code Hash: " + toHexString(this.getCodeHash());
return ret;
}
}
5 changes: 3 additions & 2 deletions ethereumj-core/src/main/java/org/ethereum/core/Block.java
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@

import static org.ethereum.crypto.HashUtil.sha3;
import static org.ethereum.datasource.MemSizeEstimator.ByteArrayEstimator;
import static org.ethereum.util.ByteUtil.toHexString;

/**
* The block in Ethereum is the collection of relevant pieces of information
Expand Down Expand Up @@ -69,7 +70,7 @@ private Block() {
}

public Block(byte[] rawData) {
logger.debug("new from [" + Hex.toHexString(rawData) + "]");
logger.debug("new from [" + toHexString(rawData) + "]");
this.rlpEncoded = rawData;
}

Expand Down Expand Up @@ -304,7 +305,7 @@ public String toString() {
parseRLP();

toStringBuff.setLength(0);
toStringBuff.append(Hex.toHexString(this.getEncoded())).append("\n");
toStringBuff.append(toHexString(this.getEncoded())).append("\n");
toStringBuff.append("BlockData [ ");
toStringBuff.append(header.toString());

Expand Down
Loading

0 comments on commit 4430b41

Please sign in to comment.