From 1253cf62480a2e1b332a72a0248b96074b01817a Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Tue, 16 Jul 2024 21:07:50 +0800 Subject: [PATCH 01/16] feat(jvm): enhance exception handling for Java 14+ NullPointerException - update condition to check for NullPointerException instances - maintain compatibility with both pre-Java 14 and Java 14+ versions - address JEP 358 changes to NullPointerException behavior - ensure correct handling of non-empty NPE messages in Java 14+ Refs: JEP 358, JDK-8220715 --- actuator/src/main/java/org/tron/core/vm/VM.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/actuator/src/main/java/org/tron/core/vm/VM.java b/actuator/src/main/java/org/tron/core/vm/VM.java index 2150df04c64..b1d7b027601 100644 --- a/actuator/src/main/java/org/tron/core/vm/VM.java +++ b/actuator/src/main/java/org/tron/core/vm/VM.java @@ -108,7 +108,10 @@ public static void play(Program program, JumpTable jumpTable) { } catch (JVMStackOverFlowException | OutOfTimeException e) { throw e; } catch (RuntimeException e) { - if (StringUtils.isEmpty(e.getMessage())) { + // https://openjdk.org/jeps/358 + // https://bugs.openjdk.org/browse/JDK-8220715 + // since jdk 14, the NullPointerExceptions message is not empty + if (e instanceof NullPointerException || StringUtils.isEmpty(e.getMessage())) { logger.warn("Unknown Exception occurred, tx id: {}", Hex.toHexString(program.getRootTransactionId()), e); program.setRuntimeFailure(new RuntimeException("Unknown Exception")); From 551df3b50ed18826a4736b212029b6156962476b Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Thu, 25 Jul 2024 15:43:56 +0800 Subject: [PATCH 02/16] feat(json-rpc/web3ClientVersion): use java.specification.version for jdk version retrieval - use `java.specification.version` instead of parsing `java.version` - ensure consistent version reporting across different Java releases(jdk10+) --- .../org/tron/core/services/jsonrpc/TronJsonRpcImpl.java | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java index 6b90ec92283..bc0f0f0ce14 100644 --- a/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java +++ b/framework/src/main/java/org/tron/core/services/jsonrpc/TronJsonRpcImpl.java @@ -24,7 +24,6 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; -import java.util.regex.Matcher; import java.util.regex.Pattern; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -230,15 +229,11 @@ public static void handleLogsFilter(LogsFilterCapsule logsFilterCapsule) { @Override public String web3ClientVersion() { - Pattern shortVersion = Pattern.compile("(\\d\\.\\d).*"); - Matcher matcher = shortVersion.matcher(System.getProperty("java.version")); - matcher.matches(); - return String.join("/", Arrays.asList( "TRON", "v" + Version.getVersion(), System.getProperty("os.name"), - "Java" + matcher.group(1))); + "Java" + System.getProperty("java.specification.version"))); } @Override From 9f168b0207bff81981b3873d65876f1089f3d45c Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Wed, 16 Oct 2024 15:28:06 +0800 Subject: [PATCH 03/16] feat(test): remove lombok.var for jdk10+ --- .../org/tron/keystroe/CredentialsTest.java | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/framework/src/test/java/org/tron/keystroe/CredentialsTest.java b/framework/src/test/java/org/tron/keystroe/CredentialsTest.java index ce992c3443f..2642129e00a 100644 --- a/framework/src/test/java/org/tron/keystroe/CredentialsTest.java +++ b/framework/src/test/java/org/tron/keystroe/CredentialsTest.java @@ -1,6 +1,5 @@ package org.tron.keystroe; -import lombok.var; import org.junit.Assert; import org.junit.Test; import org.mockito.Mockito; @@ -11,24 +10,24 @@ public class CredentialsTest { @Test public void test_equality() { - var aObject = new Object(); - var si = Mockito.mock(SignInterface.class); - var si2 = Mockito.mock(SignInterface.class); - var si3 = Mockito.mock(SignInterface.class); - var address = "TQhZ7W1RudxFdzJMw6FvMnujPxrS6sFfmj".getBytes(); - var address2 = "TNCmcTdyrYKMtmE1KU2itzeCX76jGm5Not".getBytes(); + Object aObject = new Object(); + SignInterface si = Mockito.mock(SignInterface.class); + SignInterface si2 = Mockito.mock(SignInterface.class); + SignInterface si3 = Mockito.mock(SignInterface.class); + byte[] address = "TQhZ7W1RudxFdzJMw6FvMnujPxrS6sFfmj".getBytes(); + byte[] address2 = "TNCmcTdyrYKMtmE1KU2itzeCX76jGm5Not".getBytes(); Mockito.when(si.getAddress()).thenReturn(address); Mockito.when(si2.getAddress()).thenReturn(address); Mockito.when(si3.getAddress()).thenReturn(address2); - var aCredential = Credentials.create(si); + Credentials aCredential = Credentials.create(si); Assert.assertFalse(aObject.equals(aCredential)); Assert.assertFalse(aCredential.equals(aObject)); Assert.assertFalse(aCredential.equals(null)); - var anotherCredential = Credentials.create(si); - Assert.assertTrue(aCredential.equals(anotherCredential)); - var aCredential2 = Credentials.create(si2); + Credentials anotherCredential = Credentials.create(si); Assert.assertTrue(aCredential.equals(anotherCredential)); - var aCredential3 = Credentials.create(si3); + Credentials aCredential2 = Credentials.create(si2); + Assert.assertTrue(aCredential.equals(anotherCredential)); + Credentials aCredential3 = Credentials.create(si3); Assert.assertFalse(aCredential.equals(aCredential3)); } } From 6755869c3a790414f3d016dc1981aed15d8a05f0 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Fri, 2 Aug 2024 17:48:38 +0800 Subject: [PATCH 04/16] feat(dependencies): update dependencies for jdk17 add javax.jws-api:1.1 and javax.annotation-api:1.3.2, refs:JDK-8190378, JEP 320 remove com.carrotsearch:java-sizeof:0.0.5 bump lombok from 1.18.12 to 1.18.34, refs: [Lombok Changelog](https://projectlombok.org/changelog) bump aspectjrt from 1.18.13 to 1.9.8, refs: [AspectJ Java version compatibility](https://eclipse.dev/aspectj/doc/latest/release/JavaVersionCompatibility.html) --- build.gradle | 14 +++-- common/build.gradle | 8 ++- framework/build.gradle | 1 - .../tron/common/utils/ObjectSizeUtilTest.java | 62 ------------------- 4 files changed, 15 insertions(+), 70 deletions(-) delete mode 100644 framework/src/test/java/org/tron/common/utils/ObjectSizeUtilTest.java diff --git a/build.gradle b/build.gradle index 14b095b1795..791f034919c 100644 --- a/build.gradle +++ b/build.gradle @@ -49,10 +49,16 @@ subprojects { implementation group: 'joda-time', name: 'joda-time', version: '2.3' implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' + + // https://www.oracle.com/java/technologies/javase/11-relnote-issues.html#JDK-8190378 + implementation group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' + // for json-rpc, see https://github.com/briandilley/jsonrpc4j/issues/278 + implementation group: 'javax.jws', name: 'javax.jws-api', version: '1.1' + annotationProcessor group: 'javax.annotation', name: 'javax.annotation-api', version: '1.3.2' testImplementation group: 'junit', name: 'junit', version: '4.13.2' testImplementation "org.mockito:mockito-core:4.11.0" diff --git a/common/build.gradle b/common/build.gradle index 25709c9b7a6..173a5c1ae43 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -43,9 +43,11 @@ dependencies { api group: 'io.prometheus', name: 'simpleclient', version: '0.15.0' api group: 'io.prometheus', name: 'simpleclient_httpserver', version: '0.15.0' api group: 'io.prometheus', name: 'simpleclient_hotspot', version: '0.15.0' - api 'org.aspectj:aspectjrt:1.8.13' - api 'org.aspectj:aspectjweaver:1.8.13' - api 'org.aspectj:aspectjtools:1.8.13' + // https://openjdk.org/jeps/396, JEP 396: Strongly Encapsulate JDK Internals by Default + // https://eclipse.dev/aspectj/doc/latest/release/JavaVersionCompatibility.html + api 'org.aspectj:aspectjrt:1.9.8' + api 'org.aspectj:aspectjweaver:1.9.8' + api 'org.aspectj:aspectjtools:1.9.8' api group: 'io.github.tronprotocol', name: 'libp2p', version: '2.2.5',{ exclude group: 'io.grpc', module: 'grpc-context' exclude group: 'io.grpc', module: 'grpc-core' diff --git a/framework/build.gradle b/framework/build.gradle index 0f04685f2d8..df5c3262062 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -43,7 +43,6 @@ dependencies { implementation group: 'com.google.inject', name: 'guice', version: '4.1.0' implementation group: 'io.dropwizard.metrics', name: 'metrics-core', version: '3.1.2' implementation group: 'com.github.davidb', name: 'metrics-influxdb', version: '0.8.2' - implementation group: 'com.carrotsearch', name: 'java-sizeof', version: '0.0.5' // http implementation 'org.eclipse.jetty:jetty-server:9.4.53.v20231009' implementation 'org.eclipse.jetty:jetty-servlet:9.4.53.v20231009' diff --git a/framework/src/test/java/org/tron/common/utils/ObjectSizeUtilTest.java b/framework/src/test/java/org/tron/common/utils/ObjectSizeUtilTest.java deleted file mode 100644 index c4c72991979..00000000000 --- a/framework/src/test/java/org/tron/common/utils/ObjectSizeUtilTest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * java-tron is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * java-tron is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package org.tron.common.utils; - -import static org.junit.Assert.assertEquals; - -import org.junit.Test; - -public class ObjectSizeUtilTest { - - @Test - public void testGetObjectSize() { - - Person person = new Person(); - assertEquals(48, com.carrotsearch.sizeof.RamUsageEstimator.sizeOf(person)); - Person person1 = new Person(1, "tom", new int[]{}); - assertEquals(112, com.carrotsearch.sizeof.RamUsageEstimator.sizeOf(person1)); - - Person person2 = new Person(1, "tom", new int[]{100}); - assertEquals(120, com.carrotsearch.sizeof.RamUsageEstimator.sizeOf(person2)); - - Person person3 = new Person(1, "tom", new int[]{100, 100}); - assertEquals(120, com.carrotsearch.sizeof.RamUsageEstimator.sizeOf(person3)); - Person person4 = new Person(1, "tom", new int[]{100, 100, 100}); - assertEquals(128, com.carrotsearch.sizeof.RamUsageEstimator.sizeOf(person4)); - Person person5 = new Person(1, "tom", new int[]{100, 100, 100, 100}); - assertEquals(128, com.carrotsearch.sizeof.RamUsageEstimator.sizeOf(person5)); - Person person6 = new Person(1, "tom", new int[]{100, 100, 100, 100, 100}); - assertEquals(136, com.carrotsearch.sizeof.RamUsageEstimator.sizeOf(person6)); - - } - - class Person { - - int age; - String name; - int[] scores; - - public Person() { - } - - public Person(int age, String name, int[] scores) { - this.age = age; - this.name = name; - this.scores = scores; - } - } - -} From 930a96cdddaeef8d5fabc9aa9d7db6a84354089a Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Mon, 18 Nov 2024 12:28:14 +0800 Subject: [PATCH 05/16] feat(JVM): add jvm options for jdk17 --- build.gradle | 23 +++++++++++++++- framework/build.gradle | 14 +++++++--- gradle/jdk17/java-tron.vmoptions | 8 ++++++ plugins/build.gradle | 13 +++++++-- start.sh | 46 +++++++++++++++++++++++++++----- 5 files changed, 92 insertions(+), 12 deletions(-) create mode 100644 gradle/jdk17/java-tron.vmoptions diff --git a/build.gradle b/build.gradle index 791f034919c..e668b76a2bb 100644 --- a/build.gradle +++ b/build.gradle @@ -1,14 +1,35 @@ +import org.gradle.nativeplatform.platform.internal.Architectures allprojects { version = "1.0.0" apply plugin: "java-library" } +static def isX86() { + def arch = System.getProperty("os.arch").toLowerCase() + return Architectures.X86_64.isAlias(arch) || Architectures.X86.isAlias(arch) +} + +static def isArm64() { + def arch = System.getProperty("os.arch").toLowerCase() + return new Architectures.KnownArchitecture("arm64", "aarch64").isAlias(arch) +} + +if (isArm64() && !JavaVersion.current().is(JavaVersion.VERSION_17)) { + throw new GradleException("Java 17 is required to build Java-Tron for arm64.\n" + + " Detected version ${JavaVersion.current()}") +} + +if (isX86() && !JavaVersion.current().isJava8()) { + throw new GradleException("Java 8 is required to build Java-Tron for x86.\n" + + " Detected version ${JavaVersion.current()}") +} + subprojects { apply plugin: "jacoco" apply plugin: "maven-publish" sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' jacoco { diff --git a/framework/build.gradle b/framework/build.gradle index df5c3262062..dc7249a725e 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -203,9 +203,17 @@ def createScript(project, mainClass, name) { } } } - -applicationDistribution.from("../gradle/java-tron.vmoptions") { - into "bin" +if (JavaVersion.current().is(JavaVersion.VERSION_17)) { + applicationDistribution.from("${project.rootDir}/gradle/jdk17/java-tron.vmoptions") { + into "bin" + } +} else if (JavaVersion.current().isJava8()){ + applicationDistribution.from("${project.rootDir}/gradle/java-tron.vmoptions") { + into "bin" + } +} else { + throw new GradleException("Java 8 or Java17 is supported to build Java-Tron.\n" + + " Detected version ${JavaVersion.current()}") } //distZip { // doLast { diff --git a/gradle/jdk17/java-tron.vmoptions b/gradle/jdk17/java-tron.vmoptions new file mode 100644 index 00000000000..91accd05016 --- /dev/null +++ b/gradle/jdk17/java-tron.vmoptions @@ -0,0 +1,8 @@ +-XX:+UseZGC +-Xlog:gc*:file=gc.log:time,uptime,level,tags:filecount=50,filesize=100M +-XX:ReservedCodeCacheSize=256m +-XX:+UseCodeCacheFlushing +-XX:MetaspaceSize=256m +-XX:MaxMetaspaceSize=512m +-XX:MaxDirectMemorySize=1g +-XX:+HeapDumpOnOutOfMemoryError \ No newline at end of file diff --git a/plugins/build.gradle b/plugins/build.gradle index 01afaa01708..2a8f2641341 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -127,8 +127,17 @@ def createScript(project, mainClass, name) { } } } -applicationDistribution.from("../gradle/java-tron.vmoptions") { - into "bin" +if (JavaVersion.current().is(JavaVersion.VERSION_17)) { + applicationDistribution.from("${project.rootDir}/gradle/jdk17/java-tron.vmoptions") { + into "bin" + } +} else if (JavaVersion.current().isJava8()){ + applicationDistribution.from("${project.rootDir}/gradle/java-tron.vmoptions") { + into "bin" + } +} else { + throw new GradleException("Java 8 or Java17 is supported to build Java-Tron.\n" + + " Detected version ${JavaVersion.current()}") } createScript(project, 'org.tron.plugins.ArchiveManifest', 'ArchiveManifest') createScript(project, 'org.tron.plugins.Toolkit', 'Toolkit') diff --git a/start.sh b/start.sh index 89f13cf25a7..93352d3b5c0 100644 --- a/start.sh +++ b/start.sh @@ -355,17 +355,51 @@ startService() { exit fi - nohup $JAVACMD -Xms$JVM_MS -Xmx$JVM_MX -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -Xloggc:./gc.log \ - -XX:+PrintGCDateStamps -XX:+CMSParallelRemarkEnabled -XX:ReservedCodeCacheSize=256m -XX:+UseCodeCacheFlushing \ - -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ - -XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY -XX:+HeapDumpOnOutOfMemoryError \ - -XX:NewRatio=2 -jar \ - $JAR_NAME $FULL_START_OPT >>start.log 2>&1 & + runService checkPid echo "info: start java-tron with pid $pid on $HOSTNAME" echo "info: if you need to stop the service, execute: sh start.sh --stop" } +runService() { + arch=$(uname -m) + java_version=$($JAVACMD -version 2>&1 |awk 'NR==1{ gsub(/"/,""); print $3 }') + + if [[ "$arch" == "x86_64" || "$arch" == "amd64" ]]; then + echo "Architecture: x86_64/amd64" + if [[ $java_version =~ '1.8' ]]; then + echo 'Using required JDK8 for x86_64/amd64 architecture' + nohup $JAVACMD -Xms$JVM_MS -Xmx$JVM_MX -XX:+UseConcMarkSweepGC -XX:+PrintGCDetails -Xloggc:./gc.log \ + -XX:+PrintGCDateStamps -XX:+CMSParallelRemarkEnabled -XX:ReservedCodeCacheSize=256m -XX:+UseCodeCacheFlushing \ + -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ + -XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY -XX:+HeapDumpOnOutOfMemoryError \ + -XX:NewRatio=2 -jar \ + $JAR_NAME $FULL_START_OPT >>start.log 2>&1 & + else + echo "Error: x86_64/amd64 architecture requires JDK8. Current version: $java_version" + exit 1 + fi + elif [[ "$arch" == "aarch64" || "$arch" == "arm64" ]]; then + echo "Architecture: ARM64" + if [[ $java_version =~ '17' ]]; then + echo 'Using required JDK17 for ARM architecture' + nohup $JAVACMD -Xms$JVM_MS -Xmx$JVM_MX \ + -XX:+UseZGC \ + -Xlog:gc*:file=gc.log:time,uptime,level,tags:filecount=50,filesize=100M \ + -XX:ReservedCodeCacheSize=256m -XX:+UseCodeCacheFlushing \ + -XX:MetaspaceSize=256m -XX:MaxMetaspaceSize=512m \ + -XX:MaxDirectMemorySize=$MAX_DIRECT_MEMORY -XX:+HeapDumpOnOutOfMemoryError \ + -jar $JAR_NAME $FULL_START_OPT >>start.log 2>&1 & + else + echo "Error: ARM architecture requires JDK17. Current version: $java_version" + exit 1 + fi + else + echo "Error: Unsupported architecture: $arch" + exit 1 + fi +} + rebuildManifest() { if [[ $REBUILD_MANIFEST = false ]]; then echo 'info: disable rebuild manifest!' From 08d499b049e9a8425e162deed00a498cf8b0c63c Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Wed, 4 Dec 2024 15:03:15 +0800 Subject: [PATCH 06/16] feat(docker): support ARM architecture deployment --- docker/arm64/Dockerfile | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 docker/arm64/Dockerfile diff --git a/docker/arm64/Dockerfile b/docker/arm64/Dockerfile new file mode 100644 index 00000000000..ce3f17f2298 --- /dev/null +++ b/docker/arm64/Dockerfile @@ -0,0 +1,33 @@ +FROM arm64v8/eclipse-temurin:17 + +ENV TMP_DIR="/tron-build" +ENV BASE_DIR="/java-tron" + +RUN set -o errexit -o nounset \ + && apt-get update \ + && apt-get -y install git p7zip-full wget libtcmalloc-minimal4 \ + && echo "git clone" \ + && mkdir -p $TMP_DIR \ + && cd $TMP_DIR \ + && git clone https://github.com/halibobo1205/java-tron.git \ + && cd java-tron \ + && git checkout arch/arm64 \ + && ./gradlew clean build -x test -x check --no-daemon \ + && cd build/distributions \ + && 7za x -y java-tron-1.0.0.zip \ + && mv java-tron-1.0.0 $BASE_DIR \ + && rm -rf $TMP_DIR \ + && rm -rf ~/.gradle \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libtcmalloc_minimal.so.4" +ENV TCMALLOC_RELEASE_RATE=10 + +RUN wget -P $BASE_DIR/config https://raw.githubusercontent.com/tronprotocol/tron-deployment/master/main_net_config.conf + +COPY docker-entrypoint.sh $BASE_DIR/bin + +WORKDIR $BASE_DIR + +ENTRYPOINT ["./bin/docker-entrypoint.sh"] \ No newline at end of file From 372040f6fbf4a6062dc41b4ca6f30fabf901017f Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Wed, 20 Nov 2024 19:23:38 +0800 Subject: [PATCH 07/16] feat(db): keep leveldb and rocksdb same behaviors for db operations --- .../leveldb/LevelDbDataSourceImpl.java | 64 +++-- .../rocksdb/RocksDbDataSourceImpl.java | 232 ++++++++---------- .../db/common/iterator/RockStoreIterator.java | 7 +- .../db/common/iterator/StoreIterator.java | 5 + .../leveldb/LevelDbDataSourceImplTest.java | 156 ++++++++++++ .../leveldb/RocksDbDataSourceImplTest.java | 109 ++++++++ .../org/tron/core/db2/CheckpointV2Test.java | 14 +- .../db2/RevokingDbWithCacheNewValueTest.java | 13 +- .../org/tron/core/db2/SnapshotImplTest.java | 15 +- .../tron/core/db2/SnapshotManagerTest.java | 15 +- .../org/tron/core/db2/SnapshotRootTest.java | 27 +- 11 files changed, 460 insertions(+), 197 deletions(-) diff --git a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java index 506ecdcb6c7..25a374efc0d 100644 --- a/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/leveldb/LevelDbDataSourceImpl.java @@ -18,6 +18,7 @@ import static org.fusesource.leveldbjni.JniDBFactory.factory; import com.google.common.collect.Sets; +import com.google.common.primitives.Bytes; import java.io.File; import java.io.IOException; import java.nio.file.Files; @@ -30,18 +31,14 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; -import java.util.Objects; import java.util.Set; import java.util.concurrent.locks.ReadWriteLock; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; import java.util.stream.Stream; import java.util.stream.StreamSupport; - -import com.google.common.primitives.Bytes; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.iq80.leveldb.CompressionType; import org.iq80.leveldb.DB; import org.iq80.leveldb.DBIterator; import org.iq80.leveldb.Logger; @@ -54,6 +51,7 @@ import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.metric.DbStat; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PropUtil; import org.tron.common.utils.StorageUtils; import org.tron.core.db.common.DbSourceInter; import org.tron.core.db.common.iterator.StoreIterator; @@ -75,6 +73,7 @@ public class LevelDbDataSourceImpl extends DbStat implements DbSourceInter prefixQuery(byte[] key) { } } + @Deprecated @Override public long getTotal() throws RuntimeException { resetDbLock.readLock().lock(); @@ -378,13 +404,6 @@ public long getTotal() throws RuntimeException { } } - private void updateByBatchInner(Map rows) throws Exception { - try (WriteBatch batch = database.createWriteBatch()) { - innerBatchUpdate(rows,batch); - database.write(batch, writeOptions); - } - } - private void updateByBatchInner(Map rows, WriteOptions options) throws Exception { try (WriteBatch batch = database.createWriteBatch()) { innerBatchUpdate(rows,batch); @@ -404,30 +423,23 @@ private void innerBatchUpdate(Map rows, WriteBatch batch) { @Override public void updateByBatch(Map rows, WriteOptionsWrapper options) { - resetDbLock.readLock().lock(); - try { - updateByBatchInner(rows, options.level); - } catch (Exception e) { - try { - updateByBatchInner(rows, options.level); - } catch (Exception e1) { - throw new RuntimeException(e); - } - } finally { - resetDbLock.readLock().unlock(); - } + this.updateByBatch(rows, options.level); } @Override public void updateByBatch(Map rows) { + this.updateByBatch(rows, writeOptions); + } + + private void updateByBatch(Map rows, WriteOptions options) { resetDbLock.readLock().lock(); try { - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e) { try { - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e1) { - throw new RuntimeException(e); + throw new RuntimeException(e1); } } finally { resetDbLock.readLock().unlock(); diff --git a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java index 5c051bdc101..aa6778edb6f 100644 --- a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java @@ -125,40 +125,62 @@ public void resetDb() { } } - private boolean quitIfNotAlive() { + private void throwIfNotAlive() { if (!isAlive()) { - logger.warn("DB {} is not alive.", dataBaseName); + throw new org.iq80.leveldb.DBException("DB " + this.getDBName() + " is closed."); } - return !isAlive(); } + /** copy from {@link org.fusesource.leveldbjni.internal#checkArgNotNull} */ + private static void checkArgNotNull(Object value, String name) { + if (value == null) { + throw new IllegalArgumentException("The " + name + " argument cannot be null"); + } + } + + @Deprecated @Override public Set allKeys() throws RuntimeException { resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } + try (final RocksIterator iter = getRocksIterator()) { Set result = Sets.newHashSet(); - try (final RocksIterator iter = getRocksIterator()) { - for (iter.seekToFirst(); iter.isValid(); iter.next()) { - result.add(iter.key()); - } - return result; + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + result.add(iter.key()); } + return result; } finally { resetDbLock.readLock().unlock(); } } + @Deprecated @Override public Set allValues() throws RuntimeException { - return null; + resetDbLock.readLock().lock(); + try (final RocksIterator iter = getRocksIterator()) { + Set result = Sets.newHashSet(); + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + result.add(iter.value()); + } + return result; + } finally { + resetDbLock.readLock().unlock(); + } } + @Deprecated @Override public long getTotal() throws RuntimeException { - return 0; + resetDbLock.readLock().lock(); + try (final RocksIterator iter = getRocksIterator()) { + long total = 0; + for (iter.seekToFirst(); iter.isValid(); iter.next()) { + total++; + } + return total; + } finally { + resetDbLock.readLock().unlock(); + } } @Override @@ -168,6 +190,7 @@ public String getDBName() { @Override public void setDBName(String name) { + this.dataBaseName = name; } public boolean checkOrInitEngine() { @@ -293,9 +316,9 @@ protected void log(InfoLogLevel infoLogLevel, String logMsg) { public void putData(byte[] key, byte[] value) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return; - } + throwIfNotAlive(); + checkArgNotNull(key, "key"); + checkArgNotNull(value, "value"); database.put(key, value); } catch (RocksDBException e) { throw new RuntimeException(dataBaseName, e); @@ -308,9 +331,8 @@ public void putData(byte[] key, byte[] value) { public byte[] getData(byte[] key) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return null; - } + throwIfNotAlive(); + checkArgNotNull(key, "key"); return database.get(key); } catch (RocksDBException e) { throw new RuntimeException(dataBaseName, e); @@ -323,9 +345,8 @@ public byte[] getData(byte[] key) { public void deleteData(byte[] key) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return; - } + throwIfNotAlive(); + checkArgNotNull(key, "key"); database.delete(key); } catch (RocksDBException e) { throw new RuntimeException(dataBaseName, e); @@ -344,69 +365,39 @@ public org.tron.core.db.common.iterator.DBIterator iterator() { return new RockStoreIterator(getRocksIterator()); } - private void updateByBatchInner(Map rows) throws Exception { - if (quitIfNotAlive()) { - return; - } - try (WriteBatch batch = new WriteBatch()) { - for (Map.Entry entry : rows.entrySet()) { - if (entry.getValue() == null) { - batch.delete(entry.getKey()); - } else { - batch.put(entry.getKey(), entry.getValue()); - } - } - database.write(new WriteOptions(), batch); - } - } - private void updateByBatchInner(Map rows, WriteOptions options) throws Exception { - if (quitIfNotAlive()) { - return; - } try (WriteBatch batch = new WriteBatch()) { for (Map.Entry entry : rows.entrySet()) { + checkArgNotNull(entry.getKey(), "key"); if (entry.getValue() == null) { batch.delete(entry.getKey()); } else { batch.put(entry.getKey(), entry.getValue()); } } + throwIfNotAlive(); database.write(options, batch); } } @Override public void updateByBatch(Map rows, WriteOptionsWrapper optionsWrapper) { - resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return; - } - updateByBatchInner(rows, optionsWrapper.rocks); - } catch (Exception e) { - try { - updateByBatchInner(rows); - } catch (Exception e1) { - throw new RuntimeException(dataBaseName, e1); - } - } finally { - resetDbLock.readLock().unlock(); - } + this.updateByBatch(rows, optionsWrapper.rocks); } @Override public void updateByBatch(Map rows) { + this.updateByBatch(rows, new WriteOptions()); + } + + private void updateByBatch(Map rows, WriteOptions options) { resetDbLock.readLock().lock(); try { - if (quitIfNotAlive()) { - return; - } - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e) { try { - updateByBatchInner(rows); + updateByBatchInner(rows, options); } catch (Exception e1) { throw new RuntimeException(dataBaseName, e1); } @@ -416,45 +407,34 @@ public void updateByBatch(Map rows) { } public List getKeysNext(byte[] key, long limit) { + if (limit <= 0) { + return new ArrayList<>(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return new ArrayList<>(); - } - if (limit <= 0) { - return new ArrayList<>(); - } - - try (RocksIterator iter = getRocksIterator()) { - List result = new ArrayList<>(); - long i = 0; - for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { - result.add(iter.key()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + List result = new ArrayList<>(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.add(iter.key()); } + return result; } finally { resetDbLock.readLock().unlock(); } } public Map getNext(byte[] key, long limit) { + if (limit <= 0) { + return Collections.emptyMap(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - if (limit <= 0) { - return Collections.emptyMap(); - } - try (RocksIterator iter = getRocksIterator()) { - Map result = new HashMap<>(); - long i = 0; - for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { - result.put(iter.key(), iter.value()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + Map result = new HashMap<>(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.put(iter.key(), iter.value()); } + return result; } finally { resetDbLock.readLock().unlock(); } @@ -463,78 +443,64 @@ public Map getNext(byte[] key, long limit) { @Override public Map prefixQuery(byte[] key) { resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - try (RocksIterator iterator = getRocksIterator()) { - Map result = new HashMap<>(); - for (iterator.seek(key); iterator.isValid(); iterator.next()) { - if (Bytes.indexOf(iterator.key(), key) == 0) { - result.put(WrappedByteArray.of(iterator.key()), iterator.value()); - } else { - return result; - } + try (RocksIterator iterator = getRocksIterator()) { + Map result = new HashMap<>(); + for (iterator.seek(key); iterator.isValid(); iterator.next()) { + if (Bytes.indexOf(iterator.key(), key) == 0) { + result.put(WrappedByteArray.of(iterator.key()), iterator.value()); + } else { + return result; } - return result; } + return result; } finally { resetDbLock.readLock().unlock(); } } public Set getlatestValues(long limit) { + if (limit <= 0) { + return Sets.newHashSet(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - if (limit <= 0) { - return Sets.newHashSet(); - } - try (RocksIterator iter = getRocksIterator()) { - Set result = Sets.newHashSet(); - long i = 0; - for (iter.seekToLast(); iter.isValid() && i < limit; iter.prev(), i++) { - result.add(iter.value()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + Set result = Sets.newHashSet(); + long i = 0; + for (iter.seekToLast(); iter.isValid() && i < limit; iter.prev(), i++) { + result.add(iter.value()); } + return result; } finally { resetDbLock.readLock().unlock(); } } - public Set getValuesNext(byte[] key, long limit) { + if (limit <= 0) { + return Sets.newHashSet(); + } resetDbLock.readLock().lock(); - try { - if (quitIfNotAlive()) { - return null; - } - if (limit <= 0) { - return Sets.newHashSet(); - } - try (RocksIterator iter = getRocksIterator()) { - Set result = Sets.newHashSet(); - long i = 0; - for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { - result.add(iter.value()); - } - return result; + try (RocksIterator iter = getRocksIterator()) { + Set result = Sets.newHashSet(); + long i = 0; + for (iter.seek(key); iter.isValid() && i < limit; iter.next(), i++) { + result.add(iter.value()); } + return result; } finally { resetDbLock.readLock().unlock(); } } public void backup(String dir) throws RocksDBException { + throwIfNotAlive(); Checkpoint cp = Checkpoint.create(database); cp.createCheckpoint(dir + this.getDBName()); } private RocksIterator getRocksIterator() { try ( ReadOptions readOptions = new ReadOptions().setFillCache(false)) { + throwIfNotAlive(); return database.newIterator(readOptions); } } diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java index 541f71348af..e572c201501 100644 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/RockStoreIterator.java @@ -47,7 +47,7 @@ public boolean hasNext() { try { close(); } catch (Exception e1) { - logger.error(e.getMessage(), e); + logger.error(e.getMessage(), e1); } } return hasNext; @@ -79,6 +79,11 @@ public byte[] setValue(byte[] value) { }; } + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + @Override public void seek(byte[] key) { checkState(); diff --git a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java index d771716a7e8..fccf55b1ab0 100755 --- a/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java +++ b/chainbase/src/main/java/org/tron/core/db/common/iterator/StoreIterator.java @@ -46,6 +46,11 @@ public boolean hasNext() { } } catch (Exception e) { logger.error(e.getMessage(), e); + try { + close(); + } catch (Exception e1) { + logger.error(e.getMessage(), e1); + } } return hasNext; diff --git a/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java b/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java index bf18b988f19..e717c6c7795 100644 --- a/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java +++ b/framework/src/test/java/org/tron/common/storage/leveldb/LevelDbDataSourceImplTest.java @@ -28,6 +28,7 @@ import com.google.common.collect.Sets; import java.io.File; import java.io.IOException; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -38,15 +39,24 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.iq80.leveldb.DBException; import org.junit.AfterClass; import org.junit.Assert; import org.junit.Before; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDB; +import org.tron.common.parameter.CommonParameter; +import org.tron.common.storage.WriteOptionsWrapper; +import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.PropUtil; import org.tron.common.utils.PublicMethod; +import org.tron.common.utils.StorageUtils; import org.tron.core.Constant; import org.tron.core.config.args.Args; import org.tron.core.db2.common.WrappedByteArray; @@ -73,6 +83,14 @@ public class LevelDbDataSourceImplTest { private byte[] key5 = "00000005aa".getBytes(); private byte[] key6 = "00000006aa".getBytes(); + + @Rule + public final ExpectedException exception = ExpectedException.none(); + + static { + RocksDB.loadLibrary(); + } + /** * Release resources. */ @@ -102,8 +120,19 @@ public void testPutGet() { assertNotNull(dataSourceTest.getData(key)); assertEquals(1, dataSourceTest.allKeys().size()); + assertEquals(1, dataSourceTest.getTotal()); + assertEquals(1, dataSourceTest.allValues().size()); assertEquals("50000", ByteArray.toStr(dataSourceTest.getData(key1.getBytes()))); + dataSourceTest.deleteData(key); + assertNull(dataSourceTest.getData(key)); + assertEquals(0, dataSourceTest.getTotal()); + dataSourceTest.iterator().forEachRemaining(entry -> Assert.fail("iterator should be empty")); + dataSourceTest.stream().forEach(entry -> Assert.fail("stream should be empty")); + dataSourceTest.stat(); dataSourceTest.closeDB(); + dataSourceTest.stat(); // stat again + exception.expect(DBException.class); + dataSourceTest.deleteData(key); } @Test @@ -142,6 +171,23 @@ public void testupdateByBatchInner() { assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); assertEquals("10000", ByteArray.toStr(dataSource.getData(key2.getBytes()))); assertEquals(2, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), null); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows, WriteOptionsWrapper.getInstance()); + assertEquals(0, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), value1.getBytes()); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows); + assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); + assertEquals(1, dataSource.allKeys().size()); + rows.clear(); + rows.put(null, null); + exception.expect(RuntimeException.class); + dataSource.updateByBatch(rows); dataSource.closeDB(); } @@ -354,6 +400,116 @@ public void initDbTest() { assertEquals(TronError.ErrCode.LEVELDB_INIT, thrown.getErrCode()); } + @Test + public void testCheckOrInitEngine() { + String dir = + Args.getInstance().getOutputDirectory() + Args.getInstance().getStorage().getDbDirectory(); + String enginePath = dir + File.separator + "test_engine" + File.separator + "engine.properties"; + FileUtil.createDirIfNotExists(dir + File.separator + "test_engine"); + FileUtil.createFileIfNotExists(enginePath); + PropUtil.writeProperty(enginePath, "ENGINE", "LEVELDB"); + Assert.assertEquals("LEVELDB", PropUtil.readProperty(enginePath, "ENGINE")); + + LevelDbDataSourceImpl dataSource; + dataSource = new LevelDbDataSourceImpl(dir, "test_engine"); + dataSource.initDB(); + dataSource.closeDB(); + + System.gc(); + PropUtil.writeProperty(enginePath, "ENGINE", "ROCKSDB"); + Assert.assertEquals("ROCKSDB", PropUtil.readProperty(enginePath, "ENGINE")); + try { + dataSource = new LevelDbDataSourceImpl(dir, "test_engine"); + dataSource.initDB(); + } catch (Exception e) { + Assert.assertEquals(String.format("failed to check database: %s, engine do not match", + "test_engine"), + e.getMessage()); + } + } + + @Test + public void testLevelDbOpenRocksDb() { + String name = "test_openRocksDb"; + String output = Paths + .get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); + RocksDbDataSourceImpl rocksDb = new RocksDbDataSourceImpl(output, name); + rocksDb.initDB(); + rocksDb.putData(key1, value1); + rocksDb.closeDB(); + LevelDbDataSourceImpl levelDB = + new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name), name); + exception.expectMessage(String.format("failed to check database: %s, engine do not match", + name)); + levelDB.initDB(); + } + + @Test + public void testNewInstance() { + dataSourceTest.closeDB(); + LevelDbDataSourceImpl newInst = dataSourceTest.newInstance(); + newInst.initDB(); + assertFalse(newInst.flush()); + newInst.closeDB(); + LevelDbDataSourceImpl empty = new LevelDbDataSourceImpl(); + empty.setDBName("empty"); + assertEquals("empty", empty.getDBName()); + String name = "newInst2"; + LevelDbDataSourceImpl newInst2 = new LevelDbDataSourceImpl( + StorageUtils.getOutputDirectoryByDbName(name), + name); + newInst2.initDB(); + newInst2.closeDB(); + } + + @Test + public void testGetNext() { + LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getNext_key"); + dataSource.initDB(); + dataSource.resetDb(); + putSomeKeyValue(dataSource); + // case: normal + Map seekKvLimitNext = dataSource.getNext("0000000300".getBytes(), 2); + Map hashMap = Maps.newHashMap(); + hashMap.put(ByteArray.toStr(key3), ByteArray.toStr(value3)); + hashMap.put(ByteArray.toStr(key4), ByteArray.toStr(value4)); + seekKvLimitNext.forEach((key, value) -> { + String keyStr = ByteArray.toStr(key); + Assert.assertTrue("getNext", hashMap.containsKey(keyStr)); + Assert.assertEquals(ByteArray.toStr(value), hashMap.get(keyStr)); + }); + // case: targetKey greater than all existed keys + seekKvLimitNext = dataSource.getNext("0000000700".getBytes(), 2); + Assert.assertEquals(0, seekKvLimitNext.size()); + // case: limit<=0 + seekKvLimitNext = dataSource.getNext("0000000300".getBytes(), 0); + Assert.assertEquals(0, seekKvLimitNext.size()); + dataSource.resetDb(); + dataSource.closeDB(); + } + + @Test + public void testGetlatestValues() { + LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getlatestValues_key"); + dataSource.initDB(); + dataSource.resetDb(); + putSomeKeyValue(dataSource); + // case: normal + Set seekKeyLimitNext = dataSource.getlatestValues(2); + Set hashSet = Sets.newHashSet(ByteArray.toStr(value5), ByteArray.toStr(value6)); + seekKeyLimitNext.forEach(value -> { + Assert.assertTrue(hashSet.contains(ByteArray.toStr(value))); + }); + // case: limit<=0 + seekKeyLimitNext = dataSource.getlatestValues(0); + assertEquals(0, seekKeyLimitNext.size()); + dataSource.resetDb(); + dataSource.closeDB(); + } + private void makeExceptionDb(String dbName) { LevelDbDataSourceImpl dataSource = new LevelDbDataSourceImpl( Args.getInstance().getOutputDirectory(), "test_initDb"); diff --git a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java index c6fce30e3af..621c4920029 100644 --- a/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java +++ b/framework/src/test/java/org/tron/common/storage/leveldb/RocksDbDataSourceImplTest.java @@ -10,6 +10,8 @@ import com.google.common.collect.Sets; import java.io.File; import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -20,17 +22,25 @@ import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; +import org.iq80.leveldb.DBException; import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.ClassRule; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.ExpectedException; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.common.parameter.CommonParameter; +import org.tron.common.setting.RocksDbSettings; +import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; import org.tron.common.utils.FileUtil; import org.tron.common.utils.PropUtil; import org.tron.common.utils.PublicMethod; +import org.tron.common.utils.StorageUtils; import org.tron.core.config.args.Args; import org.tron.core.db2.common.WrappedByteArray; import org.tron.core.exception.TronError; @@ -55,6 +65,9 @@ public class RocksDbDataSourceImplTest { private byte[] key5 = "00000005aa".getBytes(); private byte[] key6 = "00000006aa".getBytes(); + @Rule + public final ExpectedException expectedException = ExpectedException.none(); + /** * Release resources. */ @@ -84,8 +97,18 @@ public void testPutGet() { assertNotNull(dataSourceTest.getData(key)); assertEquals(1, dataSourceTest.allKeys().size()); + assertEquals(1, dataSourceTest.getTotal()); + assertEquals(1, dataSourceTest.allValues().size()); assertEquals("50000", ByteArray.toStr(dataSourceTest.getData(key1.getBytes()))); + dataSourceTest.deleteData(key); + assertNull(dataSourceTest.getData(key)); + assertEquals(0, dataSourceTest.getTotal()); + dataSourceTest.iterator().forEachRemaining(entry -> Assert.fail("iterator should be empty")); + dataSourceTest.stat(); dataSourceTest.closeDB(); + dataSourceTest.stat(); // stat again + expectedException.expect(DBException.class); + dataSourceTest.deleteData(key); } @Test @@ -124,6 +147,23 @@ public void testupdateByBatchInner() { assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); assertEquals("10000", ByteArray.toStr(dataSource.getData(key2.getBytes()))); assertEquals(2, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), null); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows, WriteOptionsWrapper.getInstance()); + assertEquals(0, dataSource.allKeys().size()); + + rows.clear(); + rows.put(key1.getBytes(), value1.getBytes()); + rows.put(key2.getBytes(), null); + dataSource.updateByBatch(rows); + assertEquals("50000", ByteArray.toStr(dataSource.getData(key1.getBytes()))); + assertEquals(1, dataSource.allKeys().size()); + rows.clear(); + rows.put(null, null); + expectedException.expect(RuntimeException.class); + dataSource.updateByBatch(rows); dataSource.closeDB(); } @@ -396,6 +436,75 @@ public void initDbTest() { assertEquals(TronError.ErrCode.ROCKSDB_INIT, thrown.getErrCode()); } + @Test + public void testRocksDbOpenLevelDb() { + String name = "test_openLevelDb"; + String output = Paths + .get(StorageUtils.getOutputDirectoryByDbName(name), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); + LevelDbDataSourceImpl levelDb = new LevelDbDataSourceImpl( + StorageUtils.getOutputDirectoryByDbName(name), name); + levelDb.initDB(); + levelDb.putData(key1, value1); + levelDb.closeDB(); + RocksDbDataSourceImpl rocksDb = new RocksDbDataSourceImpl(output, name); + expectedException.expectMessage( + String.format("failed to check database: %s, engine do not match", name)); + rocksDb.initDB(); + } + + @Test + public void testNewInstance() { + dataSourceTest.closeDB(); + RocksDbDataSourceImpl newInst = dataSourceTest.newInstance(); + newInst.initDB(); + assertFalse(newInst.flush()); + newInst.closeDB(); + RocksDbDataSourceImpl empty = new RocksDbDataSourceImpl(); + empty.setDBName("empty"); + assertEquals("empty", empty.getDBName()); + String output = Paths + .get(StorageUtils.getOutputDirectoryByDbName("newInst2"), CommonParameter + .getInstance().getStorage().getDbDirectory()).toString(); + RocksDbDataSourceImpl newInst2 = new RocksDbDataSourceImpl(output, "newInst2"); + newInst2.initDB(); + newInst2.closeDB(); + } + + @Test + public void backupAndDelete() throws RocksDBException { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "backupAndDelete"); + dataSource.initDB(); + putSomeKeyValue(dataSource); + Path dir = Paths.get(Args.getInstance().getOutputDirectory(), "backup"); + String path = dir + File.separator; + FileUtil.createDirIfNotExists(path); + dataSource.backup(path); + File backDB = Paths.get(dir.toString(),dataSource.getDBName()).toFile(); + Assert.assertTrue(backDB.exists()); + dataSource.deleteDbBakPath(path); + Assert.assertFalse(backDB.exists()); + dataSource.closeDB(); + } + + @Test + public void testGetTotal() { + RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( + Args.getInstance().getOutputDirectory(), "test_getTotal_key"); + dataSource.initDB(); + dataSource.resetDb(); + + Map dataMapset = Maps.newHashMap(); + dataMapset.put(key1, value1); + dataMapset.put(key2, value2); + dataMapset.put(key3, value3); + dataMapset.forEach(dataSource::putData); + Assert.assertEquals(dataMapset.size(), dataSource.getTotal()); + dataSource.resetDb(); + dataSource.closeDB(); + } + private void makeExceptionDb(String dbName) { RocksDbDataSourceImpl dataSource = new RocksDbDataSourceImpl( Args.getInstance().getOutputDirectory(), "test_initDb"); diff --git a/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java b/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java index dff2d376fd5..fb7f1987e9f 100644 --- a/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java +++ b/framework/src/test/java/org/tron/core/db2/CheckpointV2Test.java @@ -4,7 +4,7 @@ import com.google.common.primitives.Bytes; import com.google.common.primitives.Longs; import com.google.protobuf.ByteString; -import java.io.File; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -13,11 +13,12 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule; @@ -34,10 +35,12 @@ public class CheckpointV2Test { private TronApplicationContext context; private Application appT; private TestRevokingTronStore tronDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_SnapshotManager_test"}, + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); Args.getInstance().getStorage().setCheckpointVersion(2); Args.getInstance().getStorage().setCheckpointSync(true); @@ -54,9 +57,6 @@ public void removeDb() { Args.clearParam(); context.destroy(); tronDatabase.close(); - FileUtil.deleteDir(new File("output_SnapshotManager_test")); - revokingDatabase.getCheckTmpStore().close(); - tronDatabase.close(); } @Test diff --git a/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java b/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java index 2290df86978..f371f2348a7 100644 --- a/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java +++ b/framework/src/test/java/org/tron/core/db2/RevokingDbWithCacheNewValueTest.java @@ -1,22 +1,22 @@ package org.tron.core.db2; -import java.io.File; +import java.io.IOException; import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.RandomStringUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; import org.tron.common.utils.ByteArray; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.SessionOptional; import org.tron.core.Constant; import org.tron.core.capsule.utils.MarketUtils; @@ -34,12 +34,14 @@ public class RevokingDbWithCacheNewValueTest { private TronApplicationContext context; private Application appT; private TestRevokingTronStore tronDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); private String databasePath = ""; @Before - public void init() { - databasePath = "output_revokingStore_test_" + RandomStringUtils.randomAlphanumeric(10); + public void init() throws IOException { + databasePath = temporaryFolder.newFolder().toString(); Args.setParam(new String[]{"-d", databasePath}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); @@ -51,7 +53,6 @@ public void removeDb() { Args.clearParam(); context.destroy(); tronDatabase.close(); - FileUtil.deleteDir(new File(databasePath)); } @Test diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java index aab6f656b1f..649056aa151 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotImplTest.java @@ -3,16 +3,16 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; -import java.io.File; +import java.io.IOException; import java.lang.reflect.Constructor; import org.junit.After; -import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.core.Constant; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; @@ -26,10 +26,12 @@ public class SnapshotImplTest { private TronApplicationContext context; private Application appT; private SnapshotManager revokingDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF); + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); appT = ApplicationFactory.create(context); @@ -44,10 +46,7 @@ public void init() { public void removeDb() { Args.clearParam(); context.destroy(); - FileUtil.deleteDir(new File("output_revokingStore_test")); - tronDatabase.close(); - revokingDatabase.shutdown(); } /** diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java index 134dc99e51c..01bb9765aba 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java @@ -6,8 +6,7 @@ import com.google.common.collect.Maps; import com.google.common.primitives.Longs; import com.google.protobuf.ByteString; -import java.io.File; -import java.util.Arrays; +import java.io.IOException; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -15,11 +14,12 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; import org.tron.common.application.TronApplicationContext; -import org.tron.common.utils.FileUtil; import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; import org.tron.core.capsule.BlockCapsule; @@ -40,11 +40,13 @@ public class SnapshotManagerTest { private TronApplicationContext context; private Application appT; private TestRevokingTronStore tronDatabase; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_SnapshotManager_test"}, + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); appT = ApplicationFactory.create(context); @@ -59,9 +61,6 @@ public void removeDb() { Args.clearParam(); context.destroy(); tronDatabase.close(); - FileUtil.deleteDir(new File("output_SnapshotManager_test")); - revokingDatabase.getCheckTmpStore().close(); - tronDatabase.close(); } @Test diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java index 70b4d9eff30..635cc018cc2 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotRootTest.java @@ -1,10 +1,13 @@ package org.tron.core.db2; import com.google.common.collect.Sets; -import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import lombok.AllArgsConstructor; @@ -13,7 +16,9 @@ import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; import org.springframework.util.CollectionUtils; import org.tron.common.application.Application; import org.tron.common.application.ApplicationFactory; @@ -45,11 +50,13 @@ public class SnapshotRootTest { "exchange","market_order","account-trace","contract-state","trans")); private Set allDBNames; private Set allRevokingDBNames; + @Rule + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); @Before - public void init() { - Args.setParam(new String[]{"-d", "output_revokingStore_test"}, Constant.TEST_CONF); + public void init() throws IOException { + Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); appT = ApplicationFactory.create(context); } @@ -58,7 +65,6 @@ public void init() { public void removeDb() { Args.clearParam(); context.destroy(); - FileUtil.deleteDir(new File("output_revokingStore_test")); } @Test @@ -133,7 +139,9 @@ public void testSecondCacheCheck() throws ItemNotFoundException { revokingDatabase = context.getBean(SnapshotManager.class); allRevokingDBNames = parseRevokingDBNames(context); - allDBNames = Arrays.stream(new File("output_revokingStore_test/database").list()) + Path path = Paths.get(Args.getInstance().getOutputDirectory(), + Args.getInstance().getStorage().getDbDirectory()); + allDBNames = Arrays.stream(Objects.requireNonNull(path.toFile().list())) .collect(Collectors.toSet()); if (CollectionUtils.isEmpty(allDBNames)) { throw new ItemNotFoundException("No DBs found"); @@ -152,10 +160,13 @@ public void testSecondCacheCheckAddDb() revokingDatabase = context.getBean(SnapshotManager.class); allRevokingDBNames = parseRevokingDBNames(context); allRevokingDBNames.add("secondCheckTestDB"); - FileUtil.createDirIfNotExists("output_revokingStore_test/database/secondCheckTestDB"); - allDBNames = Arrays.stream(new File("output_revokingStore_test/database").list()) + Path path = Paths.get(Args.getInstance().getOutputDirectory(), + Args.getInstance().getStorage().getDbDirectory()); + Path secondCheckTestDB = Paths.get(path.toString(), "secondCheckTestDB"); + FileUtil.createDirIfNotExists(secondCheckTestDB.toString()); + allDBNames = Arrays.stream(Objects.requireNonNull(path.toFile().list())) .collect(Collectors.toSet()); - FileUtil.deleteDir(new File("output_revokingStore_test/database/secondCheckTestDB")); + FileUtil.deleteDir(secondCheckTestDB.toFile()); if (CollectionUtils.isEmpty(allDBNames)) { throw new ItemNotFoundException("No DBs found"); } From 3c84684714b9a25f4a47d9c48a4571eece156503 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Mon, 21 Oct 2024 17:37:14 +0800 Subject: [PATCH 08/16] feat(db): remove rocksDB compatibility with LevelDB for later updating RocksDB --- .../common/storage/rocksdb/RocksDbDataSourceImpl.java | 7 ++++++- plugins/README.md | 4 +--- plugins/src/main/java/org/tron/plugins/DbConvert.java | 8 +------- plugins/src/test/java/org/tron/plugins/DbConvertTest.java | 7 ------- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java index aa6778edb6f..bbaf6ce9faf 100644 --- a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java @@ -196,7 +196,12 @@ public void setDBName(String name) { public boolean checkOrInitEngine() { String dir = getDbPath().toString(); String enginePath = dir + File.separator + "engine.properties"; - + File currentFile = new File(dir, "CURRENT"); + if (currentFile.exists() && !Paths.get(enginePath).toFile().exists()) { + // if the CURRENT file exists, but the engine.properties file does not exist, it is LevelDB + logger.error(" You are trying to open a LevelDB database with RocksDB engine."); + return false; + } if (FileUtil.createDirIfNotExists(dir)) { if (!FileUtil.createFileIfNotExists(enginePath)) { return false; diff --git a/plugins/README.md b/plugins/README.md index 0db6f2e6143..90f8bce46e1 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -34,15 +34,13 @@ DB convert provides a helper which can convert LevelDB data to RocksDB data, par - ``: Input path for leveldb, default: output-directory/database. - ``: Output path for rocksdb, default: output-directory-dst/database. -- `--safe`: In safe mode, read data from leveldb then put into rocksdb, it's a very time-consuming procedure. If not, just change engine.properties from leveldb to rocksdb, rocksdb - is compatible with leveldb for the current version. This may not be the case in the future, default: false. - `-h | --help`: Provide the help info. ### Examples: ```shell script # full command - java -jar Toolkit.jar db convert [-h] [--safe] + java -jar Toolkit.jar db convert [-h] # examples java -jar Toolkit.jar db convert output-directory/database /tmp/database ``` diff --git a/plugins/src/main/java/org/tron/plugins/DbConvert.java b/plugins/src/main/java/org/tron/plugins/DbConvert.java index a75b235bbcf..c5e69b74418 100644 --- a/plugins/src/main/java/org/tron/plugins/DbConvert.java +++ b/plugins/src/main/java/org/tron/plugins/DbConvert.java @@ -49,13 +49,7 @@ public class DbConvert implements Callable { description = "Output path for rocksdb. Default: ${DEFAULT-VALUE}") private File dest; - @CommandLine.Option(names = {"--safe"}, - description = "In safe mode, read data from leveldb then put rocksdb." - + "If not, just change engine.properties from leveldb to rocksdb," - + "rocksdb is compatible with leveldb for current version." - + "This may not be the case in the future." - + "Default: ${DEFAULT-VALUE}") - private boolean safe; + private final boolean safe = true; @CommandLine.Option(names = {"-h", "--help"}) private boolean help; diff --git a/plugins/src/test/java/org/tron/plugins/DbConvertTest.java b/plugins/src/test/java/org/tron/plugins/DbConvertTest.java index 150e47c9f65..d4c0547b610 100644 --- a/plugins/src/test/java/org/tron/plugins/DbConvertTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbConvertTest.java @@ -15,13 +15,6 @@ public void testRun() throws IOException { Assert.assertEquals(0, cli.execute(args)); } - @Test - public void testRunWithSafe() throws IOException { - String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, - temporaryFolder.newFolder().toString(),"--safe" }; - Assert.assertEquals(0, cli.execute(args)); - } - @Test public void testHelp() { String[] args = new String[] {"db", "convert", "-h"}; From 005595a8ac6a52775f69eae32be2e9978f75c992 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Sat, 9 Nov 2024 13:24:39 +0800 Subject: [PATCH 09/16] feat(db): update leveldb and rocksdb for arm --- .../tron/common/storage/OptionsPicker.java | 15 +++ .../rocksdb/RocksDbDataSourceImpl.java | 121 +++++------------ .../org/tron/common/utils/StorageUtils.java | 10 +- .../java/org/tron/core/db/TronDatabase.java | 15 +-- .../tron/core/db/TronStoreWithRevoking.java | 15 +-- .../org/tron/core/db2/common/TxCacheDB.java | 9 +- .../store/MarketPairPriceToOrderStore.java | 20 --- common/build.gradle | 25 +--- .../tron/common/setting/RocksDbSettings.java | 66 ++++++++-- .../org/tron/core/config/args/Storage.java | 9 ++ .../java/org/tron/core/config/args/Args.java | 7 +- .../main/java/org/tron/program/DBConvert.java | 4 +- platform/build.gradle | 54 ++++++++ .../MarketOrderPriceComparatorForRocksDB.java | 32 +++++ .../common/org/tron/common/arch/Arch.java | 60 +++++++++ .../tron/common/utils/MarketComparator.java | 124 ++++++++++++++++++ .../MarketOrderPriceComparatorForLevelDB.java | 4 +- .../MarketOrderPriceComparatorForRocksDB.java | 7 +- plugins/build.gradle | 5 +- .../MarketOrderPriceComparatorForLevelDB.java | 28 ---- .../MarketOrderPriceComparatorForRockDB.java | 38 ------ .../java/org/tron/plugins/utils/DBUtils.java | 10 +- .../java/org/tron/plugins/DbLiteTest.java | 8 +- settings.gradle | 1 + 24 files changed, 426 insertions(+), 261 deletions(-) create mode 100644 chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java create mode 100644 platform/build.gradle create mode 100644 platform/src/main/java/arm/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java create mode 100644 platform/src/main/java/common/org/tron/common/arch/Arch.java create mode 100644 platform/src/main/java/common/org/tron/common/utils/MarketComparator.java rename {chainbase/src/main/java => platform/src/main/java/common}/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java (87%) rename chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForRockDB.java => platform/src/main/java/x86/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java (70%) delete mode 100644 plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java delete mode 100644 plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java diff --git a/chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java b/chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java new file mode 100644 index 00000000000..acace12b39d --- /dev/null +++ b/chainbase/src/main/java/org/tron/common/storage/OptionsPicker.java @@ -0,0 +1,15 @@ +package org.tron.common.storage; + +import org.tron.common.setting.RocksDbSettings; +import org.tron.common.utils.StorageUtils; + +public class OptionsPicker { + + protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) { + return StorageUtils.getOptionsByDbName(dbName); + } + + protected org.rocksdb.Options getOptionsByDbNameForRocksDB(String dbName) { + return RocksDbSettings.getOptionsByDbName(dbName); + } +} diff --git a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java index bbaf6ce9faf..5d293f99316 100644 --- a/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java +++ b/chainbase/src/main/java/org/tron/common/storage/rocksdb/RocksDbDataSourceImpl.java @@ -1,5 +1,6 @@ package org.tron.common.storage.rocksdb; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Sets; import com.google.common.primitives.Bytes; import java.io.File; @@ -20,10 +21,7 @@ import java.util.stream.Collectors; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.rocksdb.BlockBasedTableConfig; -import org.rocksdb.BloomFilter; import org.rocksdb.Checkpoint; -import org.rocksdb.DirectComparator; import org.rocksdb.InfoLogLevel; import org.rocksdb.Logger; import org.rocksdb.Options; @@ -31,7 +29,6 @@ import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; -import org.rocksdb.Statistics; import org.rocksdb.Status; import org.rocksdb.WriteBatch; import org.rocksdb.WriteOptions; @@ -53,36 +50,28 @@ public class RocksDbDataSourceImpl extends DbStat implements DbSourceInter, Iterable>, Instance { - ReadOptions readOpts; private String dataBaseName; private RocksDB database; + private Options options; private volatile boolean alive; private String parentPath; private ReadWriteLock resetDbLock = new ReentrantReadWriteLock(); private static final String KEY_ENGINE = "ENGINE"; private static final String ROCKSDB = "ROCKSDB"; - private DirectComparator comparator; private static final org.slf4j.Logger rocksDbLogger = LoggerFactory.getLogger(ROCKSDB); - public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings, - DirectComparator comparator) { + public RocksDbDataSourceImpl(String parentPath, String name, Options options) { this.dataBaseName = name; this.parentPath = parentPath; - this.comparator = comparator; - RocksDbSettings.setRocksDbSettings(settings); - initDB(); - } - - public RocksDbDataSourceImpl(String parentPath, String name, RocksDbSettings settings) { - this.dataBaseName = name; - this.parentPath = parentPath; - RocksDbSettings.setRocksDbSettings(settings); + this.options = options; initDB(); } + @VisibleForTesting public RocksDbDataSourceImpl(String parentPath, String name) { this.parentPath = parentPath; this.dataBaseName = name; + this.options = RocksDbSettings.getOptionsByDbName(name); } public Path getDbPath() { @@ -225,10 +214,6 @@ public void initDB() { throw new RuntimeException( String.format("failed to check database: %s, engine do not match", dataBaseName)); } - initDB(RocksDbSettings.getSettings()); - } - - public void initDB(RocksDbSettings settings) { resetDbLock.writeLock().lock(); try { if (isAlive()) { @@ -237,81 +222,40 @@ public void initDB(RocksDbSettings settings) { if (dataBaseName == null) { throw new IllegalArgumentException("No name set to the dbStore"); } + options.setLogger(new Logger(options) { + @Override + protected void log(InfoLogLevel infoLogLevel, String logMsg) { + rocksDbLogger.info("{} {}", dataBaseName, logMsg); + } + }); - try (Options options = new Options()) { - - // most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options + try { + logger.debug("Opening database {}.", dataBaseName); + final Path dbPath = getDbPath(); - // general options - if (settings.isEnableStatistics()) { - options.setStatistics(new Statistics()); - options.setStatsDumpPeriodSec(60); - } - options.setCreateIfMissing(true); - options.setIncreaseParallelism(1); - options.setLevelCompactionDynamicLevelBytes(true); - options.setMaxOpenFiles(settings.getMaxOpenFiles()); - - // general options supported user config - options.setNumLevels(settings.getLevelNumber()); - options.setMaxBytesForLevelMultiplier(settings.getMaxBytesForLevelMultiplier()); - options.setMaxBytesForLevelBase(settings.getMaxBytesForLevelBase()); - options.setMaxBackgroundCompactions(settings.getCompactThreads()); - options.setLevel0FileNumCompactionTrigger(settings.getLevel0FileNumCompactionTrigger()); - options.setTargetFileSizeMultiplier(settings.getTargetFileSizeMultiplier()); - options.setTargetFileSizeBase(settings.getTargetFileSizeBase()); - if (comparator != null) { - options.setComparator(comparator); + if (!Files.isSymbolicLink(dbPath.getParent())) { + Files.createDirectories(dbPath.getParent()); } - options.setLogger(new Logger(options) { - @Override - protected void log(InfoLogLevel infoLogLevel, String logMsg) { - rocksDbLogger.info("{} {}", dataBaseName, logMsg); - } - }); - - // table options - final BlockBasedTableConfig tableCfg; - options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); - tableCfg.setBlockSize(settings.getBlockSize()); - tableCfg.setBlockCache(RocksDbSettings.getCache()); - tableCfg.setCacheIndexAndFilterBlocks(true); - tableCfg.setPinL0FilterAndIndexBlocksInCache(true); - tableCfg.setFilter(new BloomFilter(10, false)); - - // read options - readOpts = new ReadOptions(); - readOpts = readOpts.setPrefixSameAsStart(true) - .setVerifyChecksums(false); try { - logger.debug("Opening database {}.", dataBaseName); - final Path dbPath = getDbPath(); - - if (!Files.isSymbolicLink(dbPath.getParent())) { - Files.createDirectories(dbPath.getParent()); + database = RocksDB.open(options, dbPath.toString()); + } catch (RocksDBException e) { + if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) { + logger.error("Database {} corrupted, please delete database directory({}) " + + "and restart.", dataBaseName, parentPath, e); + } else { + logger.error("Open Database {} failed", dataBaseName, e); } - - try { - database = RocksDB.open(options, dbPath.toString()); - } catch (RocksDBException e) { - if (Objects.equals(e.getStatus().getCode(), Status.Code.Corruption)) { - logger.error("Database {} corrupted, please delete database directory({}) " + - "and restart.", dataBaseName, parentPath, e); - } else { - logger.error("Open Database {} failed", dataBaseName, e); - } - throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT); - } - - alive = true; - } catch (IOException ioe) { - throw new RuntimeException( - String.format("failed to init database: %s", dataBaseName), ioe); + throw new TronError(e, TronError.ErrCode.ROCKSDB_INIT); } - logger.debug("Init DB {} done.", dataBaseName); + alive = true; + } catch (IOException ioe) { + throw new RuntimeException( + String.format("failed to init database: %s", dataBaseName), ioe); } + + logger.debug("Init DB {} done.", dataBaseName); } finally { resetDbLock.writeLock().unlock(); } @@ -516,7 +460,8 @@ public boolean deleteDbBakPath(String dir) { @Override public RocksDbDataSourceImpl newInstance() { - return new RocksDbDataSourceImpl(parentPath, dataBaseName, RocksDbSettings.getSettings()); + return new RocksDbDataSourceImpl(parentPath, dataBaseName, + this.options); } diff --git a/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java b/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java index 16df43f1534..06a131a35cc 100644 --- a/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java +++ b/chainbase/src/main/java/org/tron/common/utils/StorageUtils.java @@ -52,9 +52,15 @@ public static String getOutputDirectory() { } public static Options getOptionsByDbName(String dbName) { + Options options; if (hasProperty(dbName)) { - return getProperty(dbName).getDbOptions(); + options = getProperty(dbName).getDbOptions(); + } else { + options = CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName); } - return CommonParameter.getInstance().getStorage().newDefaultDbOptions(dbName); + if ("market_pair_price_to_order".equals(dbName)) { + options.comparator(new MarketOrderPriceComparatorForLevelDB()); + } + return options; } } diff --git a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java index d791e189eb4..8630fbdcdff 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java +++ b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java @@ -9,9 +9,9 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.WriteOptions; -import org.rocksdb.DirectComparator; import org.springframework.beans.factory.annotation.Autowired; import org.tron.common.parameter.CommonParameter; +import org.tron.common.storage.OptionsPicker; import org.tron.common.storage.WriteOptionsWrapper; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.metric.DbStatService; @@ -24,7 +24,7 @@ import org.tron.core.exception.ItemNotFoundException; @Slf4j(topic = "DB") -public abstract class TronDatabase implements ITronChainBase { +public abstract class TronDatabase extends OptionsPicker implements ITronChainBase { protected DbSourceInter dbSource; @Getter @@ -51,8 +51,7 @@ protected TronDatabase(String dbName) { String parentName = Paths.get(StorageUtils.getOutputDirectoryByDbName(dbName), CommonParameter.getInstance().getStorage().getDbDirectory()).toString(); dbSource = - new RocksDbDataSourceImpl(parentName, dbName, CommonParameter.getInstance() - .getRocksDBCustomSettings(), getDirectComparator()); + new RocksDbDataSourceImpl(parentName, dbName, getOptionsByDbNameForRocksDB(dbName)); } dbSource.initDB(); @@ -66,14 +65,6 @@ protected void init() { protected TronDatabase() { } - protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) { - return StorageUtils.getOptionsByDbName(dbName); - } - - protected DirectComparator getDirectComparator() { - return null; - } - public DbSourceInter getDbSource() { return dbSource; } diff --git a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java index 4b75ddee3a4..4952b70478d 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java +++ b/chainbase/src/main/java/org/tron/core/db/TronStoreWithRevoking.java @@ -16,9 +16,9 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.iq80.leveldb.WriteOptions; -import org.rocksdb.DirectComparator; import org.springframework.beans.factory.annotation.Autowired; import org.tron.common.parameter.CommonParameter; +import org.tron.common.storage.OptionsPicker; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.metric.DbStatService; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; @@ -38,7 +38,7 @@ @Slf4j(topic = "DB") -public abstract class TronStoreWithRevoking implements ITronChainBase { +public abstract class TronStoreWithRevoking extends OptionsPicker implements ITronChainBase { @Getter // only for unit test protected IRevokingDB revokingDB; @@ -69,22 +69,13 @@ protected TronStoreWithRevoking(String dbName) { .getInstance().getStorage().getDbDirectory()).toString(); this.db = new RocksDB( new RocksDbDataSourceImpl(parentPath, - dbName, CommonParameter.getInstance() - .getRocksDBCustomSettings(), getDirectComparator())); + dbName, getOptionsByDbNameForRocksDB(dbName))); } else { throw new RuntimeException(String.format("db engine %s is error", dbEngine)); } this.revokingDB = new Chainbase(new SnapshotRoot(this.db)); } - protected org.iq80.leveldb.Options getOptionsByDbNameForLevelDB(String dbName) { - return StorageUtils.getOptionsByDbName(dbName); - } - - protected DirectComparator getDirectComparator() { - return null; - } - protected TronStoreWithRevoking(DB db) { this.db = db; this.revokingDB = new Chainbase(new SnapshotRoot(db)); diff --git a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java index 9d2409685c9..db9516d6203 100644 --- a/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java +++ b/chainbase/src/main/java/org/tron/core/db2/common/TxCacheDB.java @@ -35,6 +35,7 @@ import org.tron.common.parameter.CommonParameter; import org.tron.common.prometheus.MetricKeys; import org.tron.common.prometheus.Metrics; +import org.tron.common.storage.OptionsPicker; import org.tron.common.storage.leveldb.LevelDbDataSourceImpl; import org.tron.common.storage.rocksdb.RocksDbDataSourceImpl; import org.tron.common.utils.ByteArray; @@ -48,7 +49,7 @@ import org.tron.core.store.DynamicPropertiesStore; @Slf4j(topic = "DB") -public class TxCacheDB implements DB, Flusher { +public class TxCacheDB extends OptionsPicker implements DB, Flusher { // > 65_536(= 2^16) blocks, that is the number of the reference block private static final long MAX_BLOCK_SIZE = 65536; @@ -106,7 +107,7 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore, if ("LEVELDB".equals(dbEngine.toUpperCase())) { this.persistentStore = new LevelDB( new LevelDbDataSourceImpl(StorageUtils.getOutputDirectoryByDbName(name), - name, StorageUtils.getOptionsByDbName(name), + name, getOptionsByDbNameForLevelDB(name), new WriteOptions().sync(CommonParameter.getInstance() .getStorage().isDbSync()))); } else if ("ROCKSDB".equals(dbEngine.toUpperCase())) { @@ -115,9 +116,7 @@ public TxCacheDB(String name, RecentTransactionStore recentTransactionStore, .getInstance().getStorage().getDbDirectory()).toString(); this.persistentStore = new RocksDB( - new RocksDbDataSourceImpl(parentPath, - name, CommonParameter.getInstance() - .getRocksDBCustomSettings())); + new RocksDbDataSourceImpl(parentPath, name, getOptionsByDbNameForRocksDB(name))); } else { throw new RuntimeException(String.format("db type: %s is not supported", dbEngine)); } diff --git a/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java b/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java index 605952328ed..254f0d82673 100644 --- a/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java +++ b/chainbase/src/main/java/org/tron/core/store/MarketPairPriceToOrderStore.java @@ -3,16 +3,10 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import org.iq80.leveldb.Options; -import org.rocksdb.ComparatorOptions; -import org.rocksdb.DirectComparator; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; import org.tron.common.utils.ByteUtil; -import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; -import org.tron.common.utils.MarketOrderPriceComparatorForRockDB; -import org.tron.common.utils.StorageUtils; import org.tron.core.capsule.MarketOrderIdListCapsule; import org.tron.core.capsule.utils.MarketUtils; import org.tron.core.db.TronStoreWithRevoking; @@ -26,20 +20,6 @@ protected MarketPairPriceToOrderStore(@Value("market_pair_price_to_order") Strin super(dbName); } - @Override - protected Options getOptionsByDbNameForLevelDB(String dbName) { - Options options = StorageUtils.getOptionsByDbName(dbName); - options.comparator(new MarketOrderPriceComparatorForLevelDB()); - return options; - } - - //todo: to test later - @Override - protected DirectComparator getDirectComparator() { - ComparatorOptions comparatorOptions = new ComparatorOptions(); - return new MarketOrderPriceComparatorForRockDB(comparatorOptions); - } - @Override public MarketOrderIdListCapsule get(byte[] key) throws ItemNotFoundException { byte[] value = revokingDB.get(key); diff --git a/common/build.gradle b/common/build.gradle index 173a5c1ae43..f13f575e80f 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -6,28 +6,6 @@ version '1.0.0' sourceCompatibility = 1.8 -// Dependency versions -// --------------------------------------- -def leveldbVersion = "1.8" -// -------------------------------------- - -static def isWindows() { - return org.gradle.internal.os.OperatingSystem.current().isWindows() -} - -if (isWindows()) { - ext { - leveldbGroup = "org.ethereum" - leveldbName = "leveldbjni-all" - leveldbVersion = "1.18.3" - } -} else { - ext { - leveldbGroup = "org.fusesource.leveldbjni" - leveldbName = "leveldbjni-all" - leveldbVersion = "1.8" - } -} dependencies { api group: 'com.fasterxml.jackson.core', name: 'jackson-databind', version: '2.13.4.2' // https://github.com/FasterXML/jackson-databind/issues/3627 @@ -36,8 +14,6 @@ dependencies { api group: 'commons-codec', name: 'commons-codec', version: '1.11' api group: 'com.beust', name: 'jcommander', version: '1.78' api group: 'com.typesafe', name: 'config', version: '1.3.2' - api group: leveldbGroup, name: leveldbName, version: leveldbVersion - api group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' // https://mvnrepository.com/artifact/org.quartz-scheduler/quartz api group: 'org.quartz-scheduler', name: 'quartz', version: '2.3.2' api group: 'io.prometheus', name: 'simpleclient', version: '0.15.0' @@ -66,6 +42,7 @@ dependencies { exclude group: 'org.bouncycastle', module: 'bcutil-jdk18on' } api project(":protocol") + api project(":platform") } jacocoTestReport { diff --git a/common/src/main/java/org/tron/common/setting/RocksDbSettings.java b/common/src/main/java/org/tron/common/setting/RocksDbSettings.java index 7436150cae2..500a1f5ff0e 100644 --- a/common/src/main/java/org/tron/common/setting/RocksDbSettings.java +++ b/common/src/main/java/org/tron/common/setting/RocksDbSettings.java @@ -1,16 +1,19 @@ package org.tron.common.setting; import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.rocksdb.BlockBasedTableConfig; +import org.rocksdb.BloomFilter; +import org.rocksdb.ComparatorOptions; import org.rocksdb.LRUCache; +import org.rocksdb.Options; import org.rocksdb.RocksDB; +import org.rocksdb.Statistics; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; @Slf4j public class RocksDbSettings { - @Setter - @Getter private static RocksDbSettings rocksDbSettings; @Getter @@ -60,9 +63,9 @@ public static RocksDbSettings initCustomSettings(int levelNumber, int compactThr int blockSize, long maxBytesForLevelBase, double maxBytesForLevelMultiplier, int level0FileNumCompactionTrigger, long targetFileSizeBase, - int targetFileSizeMultiplier) { + int targetFileSizeMultiplier, int maxOpenFiles) { rocksDbSettings = new RocksDbSettings() - .withMaxOpenFiles(5000) + .withMaxOpenFiles(maxOpenFiles) .withEnableStatistics(false) .withLevelNumber(levelNumber) .withCompactThreads(compactThreads) @@ -76,16 +79,17 @@ public static RocksDbSettings initCustomSettings(int levelNumber, int compactThr } public static void loggingSettings() { - logger.info(String.format( - "level number: %d, CompactThreads: %d, Blocksize: %d, maxBytesForLevelBase: %d," - + " withMaxBytesForLevelMultiplier: %f, level0FileNumCompactionTrigger: %d, " - + "withTargetFileSizeBase: %d, withTargetFileSizeMultiplier: %d", + logger.info( + "level number: {}, CompactThreads: {}, Blocksize:{}, maxBytesForLevelBase: {}," + + " withMaxBytesForLevelMultiplier: {}, level0FileNumCompactionTrigger: {}, " + + "withTargetFileSizeBase: {}, withTargetFileSizeMultiplier: {}, maxOpenFiles: {}", rocksDbSettings.getLevelNumber(), rocksDbSettings.getCompactThreads(), rocksDbSettings.getBlockSize(), rocksDbSettings.getMaxBytesForLevelBase(), rocksDbSettings.getMaxBytesForLevelMultiplier(), rocksDbSettings.getLevel0FileNumCompactionTrigger(), - rocksDbSettings.getTargetFileSizeBase(), rocksDbSettings.getTargetFileSizeMultiplier())); + rocksDbSettings.getTargetFileSizeBase(), rocksDbSettings.getTargetFileSizeMultiplier(), + rocksDbSettings.getMaxOpenFiles()); } public RocksDbSettings withMaxOpenFiles(int maxOpenFiles) { @@ -140,4 +144,46 @@ public RocksDbSettings withTargetFileSizeMultiplier(int targetFileSizeMultiplier public static LRUCache getCache() { return cache; } + + public static Options getOptionsByDbName(String dbName) { + RocksDbSettings settings = getSettings(); + + Options options = new Options(); + + // most of these options are suggested by https://github.com/facebook/rocksdb/wiki/Set-Up-Options + + // general options + if (settings.isEnableStatistics()) { + options.setStatistics(new Statistics()); + options.setStatsDumpPeriodSec(60); + } + options.setCreateIfMissing(true); + options.setIncreaseParallelism(1); + options.setLevelCompactionDynamicLevelBytes(true); + options.setMaxOpenFiles(settings.getMaxOpenFiles()); + + // general options supported user config + options.setNumLevels(settings.getLevelNumber()); + options.setMaxBytesForLevelMultiplier(settings.getMaxBytesForLevelMultiplier()); + options.setMaxBytesForLevelBase(settings.getMaxBytesForLevelBase()); + options.setMaxBackgroundCompactions(settings.getCompactThreads()); + options.setLevel0FileNumCompactionTrigger(settings.getLevel0FileNumCompactionTrigger()); + options.setTargetFileSizeMultiplier(settings.getTargetFileSizeMultiplier()); + options.setTargetFileSizeBase(settings.getTargetFileSizeBase()); + + // table options + final BlockBasedTableConfig tableCfg; + options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); + tableCfg.setBlockSize(settings.getBlockSize()); + tableCfg.setBlockCache(RocksDbSettings.getCache()); + tableCfg.setCacheIndexAndFilterBlocks(true); + tableCfg.setPinL0FilterAndIndexBlocksInCache(true); + tableCfg.setFilter(new BloomFilter(10, false)); + if ("market_pair_price_to_order".equals(dbName)) { + ComparatorOptions comparatorOptions = new ComparatorOptions(); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(comparatorOptions)); + } + + return options; + } } diff --git a/common/src/main/java/org/tron/core/config/args/Storage.java b/common/src/main/java/org/tron/core/config/args/Storage.java index 9cf6eb6bab1..655b6b779fe 100644 --- a/common/src/main/java/org/tron/core/config/args/Storage.java +++ b/common/src/main/java/org/tron/core/config/args/Storage.java @@ -25,9 +25,11 @@ import java.util.stream.Collectors; import lombok.Getter; import lombok.Setter; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.iq80.leveldb.CompressionType; import org.iq80.leveldb.Options; +import org.tron.common.arch.Arch; import org.tron.common.cache.CacheStrategies; import org.tron.common.cache.CacheType; import org.tron.common.utils.DbOptionalsUtils; @@ -42,6 +44,7 @@ * @version 1.0 * @since 2018/5/25 */ +@Slf4j(topic = "db") public class Storage { /** @@ -87,6 +90,7 @@ public class Storage { * Default values of directory */ private static final String DEFAULT_DB_ENGINE = "LEVELDB"; + private static final String ROCKS_DB_ENGINE = "ROCKSDB"; private static final boolean DEFAULT_DB_SYNC = false; private static final boolean DEFAULT_EVENT_SUBSCRIBE_CONTRACT_PARSE = true; private static final String DEFAULT_DB_DIRECTORY = "database"; @@ -171,6 +175,11 @@ public class Storage { private final Map dbRoots = Maps.newConcurrentMap(); public static String getDbEngineFromConfig(final Config config) { + if (Arch.isArm64()) { + // if is arm64 but config is leveldb, should throw exception? + logger.warn("Arm64 architecture detected, using RocksDB as db engine, ignore config."); + return ROCKS_DB_ENGINE; + } return config.hasPath(DB_ENGINE_CONFIG_KEY) ? config.getString(DB_ENGINE_CONFIG_KEY) : DEFAULT_DB_ENGINE; } diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index edf82c3068c..64dffa8843e 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -45,6 +45,7 @@ import org.quartz.CronExpression; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; +import org.tron.common.arch.Arch; import org.tron.common.args.Account; import org.tron.common.args.GenesisBlock; import org.tron.common.args.Witness; @@ -1667,11 +1668,13 @@ private static void initRocksDbSettings(Config config) { .getLong(prefix + "targetFileSizeBase") : 64; int targetFileSizeMultiplier = config.hasPath(prefix + "targetFileSizeMultiplier") ? config .getInt(prefix + "targetFileSizeMultiplier") : 1; + int maxOpenFiles = config.hasPath(prefix + "maxOpenFiles") + ? config.getInt(prefix + "maxOpenFiles") : 5000; PARAMETER.rocksDBCustomSettings = RocksDbSettings .initCustomSettings(levelNumber, compactThreads, blocksize, maxBytesForLevelBase, maxBytesForLevelMultiplier, level0FileNumCompactionTrigger, - targetFileSizeBase, targetFileSizeMultiplier); + targetFileSizeBase, targetFileSizeMultiplier, maxOpenFiles); RocksDbSettings.loggingSettings(); } @@ -1708,6 +1711,8 @@ private static void initBackupProperty(Config config) { public static void logConfig() { CommonParameter parameter = CommonParameter.getInstance(); logger.info("\n"); + logger.info("************************ System info ************************"); + logger.info("{}", Arch.withAll()); logger.info("************************ Net config ************************"); logger.info("P2P version: {}", parameter.getNodeP2pVersion()); logger.info("LAN IP: {}", parameter.getNodeLanIp()); diff --git a/framework/src/main/java/org/tron/program/DBConvert.java b/framework/src/main/java/org/tron/program/DBConvert.java index 7b9d63544dc..d656ca82043 100644 --- a/framework/src/main/java/org/tron/program/DBConvert.java +++ b/framework/src/main/java/org/tron/program/DBConvert.java @@ -34,7 +34,7 @@ import org.rocksdb.Status; import org.tron.common.utils.FileUtil; import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; -import org.tron.common.utils.MarketOrderPriceComparatorForRockDB; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; import org.tron.common.utils.PropUtil; @Slf4j @@ -195,7 +195,7 @@ private Options newDefaultRocksDbOptions() { options.setLevel0FileNumCompactionTrigger(4); options.setLevelCompactionDynamicLevelBytes(true); if ("market_pair_price_to_order".equalsIgnoreCase(this.dbName)) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } final BlockBasedTableConfig tableCfg; options.setTableFormatConfig(tableCfg = new BlockBasedTableConfig()); diff --git a/platform/build.gradle b/platform/build.gradle new file mode 100644 index 00000000000..39e7aec0922 --- /dev/null +++ b/platform/build.gradle @@ -0,0 +1,54 @@ +import org.gradle.nativeplatform.platform.internal.Architectures + +description = "platform – a distributed consensus arithmetic for blockchain." + + +static def isX86() { + def arch = System.getProperty("os.arch").toLowerCase() + return Architectures.X86_64.isAlias(arch) || Architectures.X86.isAlias(arch) +} + +static def isArm64() { + def arch = System.getProperty("os.arch").toLowerCase() + return new Architectures.KnownArchitecture("arm64", "aarch64").isAlias(arch) +} + +if (isX86()) { + ext { + leveldbGroup = "org.fusesource.leveldbjni" + leveldbName = "leveldbjni-all" + leveldbVersion = "1.8" + rocksDBVersion = "5.15.10" + } +} else if (isArm64()) { + ext { + leveldbGroup = "com.halibobor" + leveldbName = "leveldbjni-all" + leveldbVersion = "1.18.3" + rocksDBVersion = "7.7.3" + } +} else { + throw new GradleException("Unsupported architecture: ${System.getProperty("os.arch")}") +} + +sourceSets { + x86 { + java { + srcDirs 'src/main/java/x86', 'src/main/java/common' + } + } + arm { + java { + srcDirs 'src/main/java/arm', 'src/main/java/common' + } + } +} + +dependencies { + api group: leveldbGroup, name: leveldbName, version: leveldbVersion + api group: 'org.rocksdb', name: 'rocksdbjni', version: rocksDBVersion +} + +tasks.withType(JavaCompile).configureEach { + source = isX86() ? sourceSets.x86.java : sourceSets.arm.java +} diff --git a/platform/src/main/java/arm/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java b/platform/src/main/java/arm/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java new file mode 100644 index 00000000000..26c246faf0e --- /dev/null +++ b/platform/src/main/java/arm/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java @@ -0,0 +1,32 @@ +package org.tron.common.utils; + +import java.nio.ByteBuffer; +import org.rocksdb.AbstractComparator; +import org.rocksdb.ComparatorOptions; + +public class MarketOrderPriceComparatorForRocksDB extends AbstractComparator { + + public MarketOrderPriceComparatorForRocksDB(final ComparatorOptions copt) { + super(copt); + } + + @Override + public String name() { + return "MarketOrderPriceComparator"; + } + + @Override + public int compare(final ByteBuffer a, final ByteBuffer b) { + return MarketComparator.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); + } + + /** + * DirectSlice.data().array will throw UnsupportedOperationException. + * */ + public byte[] convertDataToBytes(ByteBuffer buf) { + byte[] bytes = new byte[buf.remaining()]; + buf.get(bytes); + return bytes; + } + +} diff --git a/platform/src/main/java/common/org/tron/common/arch/Arch.java b/platform/src/main/java/common/org/tron/common/arch/Arch.java new file mode 100644 index 00000000000..3717fc1a016 --- /dev/null +++ b/platform/src/main/java/common/org/tron/common/arch/Arch.java @@ -0,0 +1,60 @@ +package org.tron.common.arch; + +public final class Arch { + + private Arch() { + } + + public static String withAll() { + final StringBuilder info = new StringBuilder(); + info.append("os.name").append(": ").append(getOsName()).append("\n"); + info.append("os.arch").append(": ").append(getOsArch()).append("\n"); + info.append("bit.model").append(": ").append(getBitModel()).append("\n"); + info.append("java.version").append(": ").append(javaVersion()).append("\n"); + info.append("java.specification.version").append(": ").append(javaSpecificationVersion()) + .append("\n"); + info.append("java.vendor").append(": ").append(javaVendor()).append("\n"); + return info.toString(); + } + + public static String getOsName() { + return System.getProperty("os.name").toLowerCase().trim(); + + } + public static String getOsArch() { + return System.getProperty("os.arch").toLowerCase().trim(); + } + + public static int getBitModel() { + String prop = System.getProperty("sun.arch.data.model"); + if (prop == null) { + prop = System.getProperty("com.ibm.vm.bitmode"); + } + if (prop != null) { + return Integer.parseInt(prop); + } + // GraalVM support, see https://github.com/fusesource/jansi/issues/162 + String arch = System.getProperty("os.arch"); + if (arch.endsWith("64") && "Substrate VM".equals(System.getProperty("java.vm.name"))) { + return 64; + } + return -1; // we don't know... + } + + public static String javaVersion() { + return System.getProperty("java.version").toLowerCase().trim(); + } + + public static String javaSpecificationVersion() { + return System.getProperty("java.specification.version").toLowerCase().trim(); + } + + public static String javaVendor() { + return System.getProperty("java.vendor").toLowerCase().trim(); + } + + public static boolean isArm64() { + String osArch = getOsArch(); + return osArch.contains("arm64") || osArch.contains("aarch64"); + } +} diff --git a/platform/src/main/java/common/org/tron/common/utils/MarketComparator.java b/platform/src/main/java/common/org/tron/common/utils/MarketComparator.java new file mode 100644 index 00000000000..c2742d4d10b --- /dev/null +++ b/platform/src/main/java/common/org/tron/common/utils/MarketComparator.java @@ -0,0 +1,124 @@ +package org.tron.common.utils; + +import java.math.BigInteger; + +public class MarketComparator { + + public static final int TOKEN_ID_LENGTH = Long.toString(Long.MAX_VALUE).getBytes().length; // 19 + + + public static int comparePriceKey(byte[] o1, byte[] o2) { + //compare pair + byte[] pair1 = new byte[TOKEN_ID_LENGTH * 2]; + byte[] pair2 = new byte[TOKEN_ID_LENGTH * 2]; + + System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2); + System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2); + + int pairResult = compareUnsigned(pair1, pair2); + if (pairResult != 0) { + return pairResult; + } + + //compare price + byte[] getSellTokenQuantity1 = new byte[8]; + byte[] getBuyTokenQuantity1 = new byte[8]; + + byte[] getSellTokenQuantity2 = new byte[8]; + byte[] getBuyTokenQuantity2 = new byte[8]; + + int longByteNum = 8; + + System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + getSellTokenQuantity1, 0, longByteNum); + System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, + getBuyTokenQuantity1, 0, longByteNum); + + System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + getSellTokenQuantity2, 0, longByteNum); + System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, + getBuyTokenQuantity2, 0, longByteNum); + + long sellTokenQuantity1 = toLong(getSellTokenQuantity1); + long buyTokenQuantity1 = toLong(getBuyTokenQuantity1); + long sellTokenQuantity2 = toLong(getSellTokenQuantity2); + long buyTokenQuantity2 = toLong(getBuyTokenQuantity2); + + if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) + && (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) { + return 0; + } + + if (sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) { + return -1; + } + + if (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0) { + return 1; + } + + return comparePrice(sellTokenQuantity1, buyTokenQuantity1, + sellTokenQuantity2, buyTokenQuantity2); + + } + + /** + * Note: the params should be the same token pair, or you should change the order. + * All the quantity should be bigger than 0. + * */ + public static int comparePrice(long price1SellQuantity, long price1BuyQuantity, + long price2SellQuantity, long price2BuyQuantity) { + try { + return Long.compare(StrictMath.multiplyExact(price1BuyQuantity, price2SellQuantity), + StrictMath.multiplyExact(price2BuyQuantity, price1SellQuantity)); + + } catch (ArithmeticException ex) { + // do nothing here, because we will use BigInteger to compute again + } + + BigInteger price1BuyQuantityBI = BigInteger.valueOf(price1BuyQuantity); + BigInteger price1SellQuantityBI = BigInteger.valueOf(price1SellQuantity); + BigInteger price2BuyQuantityBI = BigInteger.valueOf(price2BuyQuantity); + BigInteger price2SellQuantityBI = BigInteger.valueOf(price2SellQuantity); + + return price1BuyQuantityBI.multiply(price2SellQuantityBI) + .compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI)); + } + + /** + * copy from org.bouncycastle.util.Arrays.compareUnsigned + */ + private static int compareUnsigned(byte[] a, byte[] b) { + if (a == b) { + return 0; + } + if (a == null) { + return -1; + } + if (b == null) { + return 1; + } + int minLen = StrictMath.min(a.length, b.length); + for (int i = 0; i < minLen; ++i) { + int aVal = a[i] & 0xFF; + int bVal = b[i] & 0xFF; + if (aVal < bVal) { + return -1; + } + if (aVal > bVal) { + return 1; + } + } + if (a.length < b.length) { + return -1; + } + if (a.length > b.length) { + return 1; + } + return 0; + } + + public static long toLong(byte[] b) { + return (b == null || b.length == 0) ? 0 : new BigInteger(1, b).longValue(); + } +} diff --git a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java b/platform/src/main/java/common/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java similarity index 87% rename from chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java rename to platform/src/main/java/common/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java index c4c519f68f1..efb7219b211 100644 --- a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java +++ b/platform/src/main/java/common/org/tron/common/utils/MarketOrderPriceComparatorForLevelDB.java @@ -1,7 +1,5 @@ package org.tron.common.utils; -import org.tron.core.capsule.utils.MarketUtils; - public class MarketOrderPriceComparatorForLevelDB implements org.iq80.leveldb.DBComparator { @Override @@ -26,7 +24,7 @@ public byte[] findShortSuccessor(byte[] key) { */ @Override public int compare(byte[] o1, byte[] o2) { - return MarketUtils.comparePriceKey(o1, o2); + return MarketComparator.comparePriceKey(o1, o2); } } diff --git a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForRockDB.java b/platform/src/main/java/x86/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java similarity index 70% rename from chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForRockDB.java rename to platform/src/main/java/x86/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java index d3812f0f6bc..be406ff658d 100644 --- a/chainbase/src/main/java/org/tron/common/utils/MarketOrderPriceComparatorForRockDB.java +++ b/platform/src/main/java/x86/org/tron/common/utils/MarketOrderPriceComparatorForRocksDB.java @@ -3,11 +3,10 @@ import org.rocksdb.ComparatorOptions; import org.rocksdb.DirectSlice; import org.rocksdb.util.DirectBytewiseComparator; -import org.tron.core.capsule.utils.MarketUtils; -public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { +public class MarketOrderPriceComparatorForRocksDB extends DirectBytewiseComparator { - public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { + public MarketOrderPriceComparatorForRocksDB(final ComparatorOptions copt) { super(copt); } @@ -18,7 +17,7 @@ public String name() { @Override public int compare(final DirectSlice a, final DirectSlice b) { - return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); + return MarketComparator.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); } /** diff --git a/plugins/build.gradle b/plugins/build.gradle index 2a8f2641341..7745ccf3f79 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -29,10 +29,9 @@ dependencies { implementation group: 'com.typesafe', name: 'config', version: '1.3.2' implementation group: 'me.tongfei', name: 'progressbar', version: '0.9.3' implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - implementation group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' - implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' - implementation 'io.github.tronprotocol:leveldb:1.18.2' + implementation 'com.halibobor:leveldb:1.18.3' implementation project(":protocol") + implementation(project(":platform")) } check.dependsOn 'lint' diff --git a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java deleted file mode 100644 index 0879f770e1f..00000000000 --- a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.tron.plugins.comparator; - -import org.iq80.leveldb.DBComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForLevelDB implements DBComparator { - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public byte[] findShortestSeparator(byte[] start, byte[] limit) { - return new byte[0]; - } - - @Override - public byte[] findShortSuccessor(byte[] key) { - return new byte[0]; - } - - @Override - public int compare(byte[] o1, byte[] o2) { - return MarketUtils.comparePriceKey(o1, o2); - } - -} \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java b/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java deleted file mode 100644 index cd718bdd2d7..00000000000 --- a/plugins/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.tron.plugins.comparator; - -import org.rocksdb.ComparatorOptions; -import org.rocksdb.DirectSlice; -import org.rocksdb.util.DirectBytewiseComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { - - public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { - super(copt); - } - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public int compare(final DirectSlice a, final DirectSlice b) { - return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); - } - - /** - * DirectSlice.data().array will throw UnsupportedOperationException. - * */ - public byte[] convertDataToBytes(DirectSlice directSlice) { - int capacity = directSlice.data().capacity(); - byte[] bytes = new byte[capacity]; - - for (int i = 0; i < capacity; i++) { - bytes[i] = directSlice.get(i); - } - - return bytes; - } - -} \ No newline at end of file diff --git a/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java b/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java index f8559d5dba8..e572159913d 100644 --- a/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java +++ b/plugins/src/main/java/org/tron/plugins/utils/DBUtils.java @@ -16,8 +16,8 @@ import org.rocksdb.Options; import org.rocksdb.RocksDB; import org.rocksdb.RocksDBException; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForLevelDB; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForRockDB; +import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; import org.tron.protos.Protocol; public class DBUtils { @@ -115,7 +115,7 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { public static RocksDB newRocksDb(Path db) throws RocksDBException { try (Options options = newDefaultRocksDbOptions(false)) { if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } return RocksDB.open(options, db.toString()); } @@ -124,7 +124,7 @@ public static RocksDB newRocksDb(Path db) throws RocksDBException { public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { try (Options options = newDefaultRocksDbOptions(true)) { if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } return RocksDB.open(options, db.toString()); } @@ -134,7 +134,7 @@ public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { public static RocksDB newRocksDbReadOnly(Path db) throws RocksDBException { try (Options options = newDefaultRocksDbOptions(false)) { if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } return RocksDB.openReadOnly(options, db.toString()); } diff --git a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java index b4c66c9820f..af87f74bd77 100644 --- a/plugins/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/plugins/src/test/java/org/tron/plugins/DbLiteTest.java @@ -67,9 +67,10 @@ public void shutdown() throws InterruptedException { context.close(); } - public void init() throws IOException { + public void init(String dbType) throws IOException { dbPath = folder.newFolder().toString(); - Args.setParam(new String[]{"-d", dbPath, "-w", "--p2p-disable", "true"}, + Args.setParam(new String[]{ + "-d", dbPath, "-w", "--p2p-disable", "true", "--storage-db-engine", dbType}, "config-localtest.conf"); // allow account root Args.getInstance().setAllowAccountStateRoot(1); @@ -89,7 +90,7 @@ void testTools(String dbType, int checkpointVersion) throws InterruptedException, IOException { logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); - init(); + init(dbType); final String[] argsForSnapshot = new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", @@ -101,7 +102,6 @@ void testTools(String dbType, int checkpointVersion) final String[] argsForMerge = new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath + File.separator + "history"}; - Args.getInstance().getStorage().setDbEngine(dbType); Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); DbLite.setRecentBlks(3); // start fullNode diff --git a/settings.gradle b/settings.gradle index eb304444378..af32bfca702 100644 --- a/settings.gradle +++ b/settings.gradle @@ -8,4 +8,5 @@ include 'common' include 'example:actuator-example' include 'crypto' include 'plugins' +include 'platform' From 2874980efe445186a12fa9d2b63a51d34e068f51 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Fri, 14 Mar 2025 16:30:46 +0800 Subject: [PATCH 10/16] feat(math): add hard-code for arm pow --- .../capsule/utils/ExchangeProcessorTest.java | 65 ++++- .../tron/core/db2/SnapshotManagerTest.java | 1 + .../arm/org/tron/common/math/MathWrapper.java | 252 ++++++++++++++++++ .../org/tron/common/math/MathWrapper.java | 0 4 files changed, 311 insertions(+), 7 deletions(-) create mode 100644 platform/src/main/java/arm/org/tron/common/math/MathWrapper.java rename {common/src/main/java => platform/src/main/java/x86}/org/tron/common/math/MathWrapper.java (100%) diff --git a/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java b/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java index 717c62b01a8..1f0be4b1f7c 100644 --- a/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java +++ b/framework/src/test/java/org/tron/core/capsule/utils/ExchangeProcessorTest.java @@ -138,12 +138,63 @@ public void testWithdraw() { @Test public void testStrictMath() { long supply = 1_000_000_000_000_000_000L; - ExchangeProcessor processor = new ExchangeProcessor(supply, false); - long anotherTokenQuant = processor.exchange(4732214, 2202692725330L, 29218); - processor = new ExchangeProcessor(supply, true); - long result = processor.exchange(4732214, 2202692725330L, 29218); - Assert.assertNotEquals(anotherTokenQuant, result); + long[][] testData = { + {4732214L, 2202692725330L, 29218L}, + {5618633L, 556559904655L, 1L}, + {9299554L, 1120271441185L, 7000L}, + {62433133L, 12013267997895L, 100000L}, + {64212664L, 725836766395L, 50000L}, + {64126212L, 2895100109660L, 5000L}, + {56459055L, 3288380567368L, 165000L}, + {21084707L, 1589204008960L, 50000L}, + {24120521L, 1243764649177L, 20000L}, + {836877L, 212532333234L, 5293L}, + {55879741L, 13424854054078L, 250000L}, + {66388882L, 11300012790454L, 300000L}, + {94470955L, 7941038150919L, 2000L}, + {13613746L, 5012660712983L, 122L}, + {71852829L, 5262251868618L, 396L}, + {3857658L, 446109245044L, 20637L}, + {35491863L, 3887393269796L, 100L}, + {295632118L, 1265298439004L, 500000L}, + {49320113L, 1692106302503L, 123267L}, + {10966984L, 6222910652894L, 2018L}, + {41634280L, 2004508994767L, 865L}, + {10087714L, 6765558834714L, 1009L}, + {42270078L, 210360843525L, 200000L}, + {571091915L, 655011397250L, 2032520L}, + {51026781L, 1635726339365L, 37L}, + {61594L, 312318864132L, 500L}, + {11616684L, 5875978057357L, 20L}, + {60584529L, 1377717821301L, 78132L}, + {29818073L, 3033545989651L, 182L}, + {3855280L, 834647482043L, 16L}, + {58310711L, 1431562205655L, 200000L}, + {60226263L, 1386036785882L, 178226L}, + {3537634L, 965771433992L, 225L}, + {3760534L, 908700758784L, 328L}, + {80913L, 301864126445L, 4L}, + {3789271L, 901842209723L, 1L}, + {4051904L, 843419481286L, 1005L}, + {89141L, 282107742510L, 100L}, + {90170L, 282854635378L, 26L}, + {4229852L, 787503315944L, 137L}, + {4259884L, 781975090197L, 295L}, + {3627657L, 918682223700L, 34L}, + {813519L, 457546358759L, 173L}, + {89626L, 327856173057L, 27L}, + {97368L, 306386489550L, 50L}, + {93712L, 305866015731L, 4L}, + {3281260L, 723656594544L, 40L}, + {3442652L, 689908773685L, 18L}, + }; + + for (long[] data : testData) { + ExchangeProcessor processor = new ExchangeProcessor(supply, false); + long anotherTokenQuant = processor.exchange(data[0], data[1], data[2]); + processor = new ExchangeProcessor(supply, true); + long result = processor.exchange(data[0], data[1], data[2]); + Assert.assertNotEquals(anotherTokenQuant, result); + } } - - } diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java index 01bb9765aba..d6fd319e6a0 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java @@ -7,6 +7,7 @@ import com.google.common.primitives.Longs; import com.google.protobuf.ByteString; import java.io.IOException; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; diff --git a/platform/src/main/java/arm/org/tron/common/math/MathWrapper.java b/platform/src/main/java/arm/org/tron/common/math/MathWrapper.java new file mode 100644 index 00000000000..81f269a3026 --- /dev/null +++ b/platform/src/main/java/arm/org/tron/common/math/MathWrapper.java @@ -0,0 +1,252 @@ +package org.tron.common.math; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +/** + * This class is deprecated and should not be used in new code, + * for cross-platform consistency, please use {@link StrictMathWrapper} instead, + * especially for floating-point calculations. + */ +@Deprecated +public class MathWrapper { + + private static final Map powData = Collections.synchronizedMap(new HashMap<>()); + private static final String POW = "3f40624dd2f1a9fc"; // 1/2000 = 0.0005 + + public static double pow(double a, double b) { + double strictResult = StrictMath.pow(a, b); + return powData.getOrDefault(new PowData(a, b), strictResult); + } + + /** + * This static block is used to initialize the data map. + */ + static { + addPowData("3ff0192278704be3", POW, "3ff000033518c576"); // 4137160 + addPowData("3ff000002fc6a33f", POW, "3ff0000000061d86"); // 4065476 + addPowData("3ff00314b1e73ecf", POW, "3ff0000064ea3ef8"); // 4071538 + addPowData("3ff0068cd52978ae", POW, "3ff00000d676966c"); // 4109544 + addPowData("3ff0032fda05447d", POW, "3ff0000068636fe0"); // 4123826 + addPowData("3ff00051c09cc796", POW, "3ff000000a76c20e"); // 4166806 + addPowData("3ff00bef8115b65d", POW, "3ff0000186893de0"); // 4225778 + addPowData("3ff009b0b2616930", POW, "3ff000013d27849e"); // 4251796 + addPowData("3ff00364ba163146", POW, "3ff000006f26a9dc"); // 4257157 + addPowData("3ff019be4095d6ae", POW, "3ff0000348e9f02a"); // 4260583 + addPowData("3ff0123e52985644", POW, "3ff0000254797fd0"); // 4367125 + addPowData("3ff0126d052860e2", POW, "3ff000025a6cde26"); // 4402197 + addPowData("3ff0001632cccf1b", POW, "3ff0000002d76406"); // 4405788 + addPowData("3ff0000965922b01", POW, "3ff000000133e966"); // 4490332 + addPowData("3ff00005c7692d61", POW, "3ff0000000bd5d34"); // 4499056 + addPowData("3ff015cba20ec276", POW, "3ff00002c84cef0e"); // 4518035 + addPowData("3ff00002f453d343", POW, "3ff000000060cf4e"); // 4533215 + addPowData("3ff006ea73f88946", POW, "3ff00000e26d4ea2"); // 4647814 + addPowData("3ff00a3632db72be", POW, "3ff000014e3382a6"); // 4766695 + addPowData("3ff000c0e8df0274", POW, "3ff0000018b0aeb2"); // 4771494 + addPowData("3ff00015c8f06afe", POW, "3ff0000002c9d73e"); // 4793587 + addPowData("3ff00068def18101", POW, "3ff000000d6c3cac"); // 4801947 + addPowData("3ff01349f3ac164b", POW, "3ff000027693328a"); // 4916843 + addPowData("3ff00e86a7859088", POW, "3ff00001db256a52"); // 4924111 + addPowData("3ff00000c2a51ab7", POW, "3ff000000018ea20"); // 5098864 + addPowData("3ff020fb74e9f170", POW, "3ff00004346fbfa2"); // 5133963 + addPowData("3ff00001ce277ce7", POW, "3ff00000003b27dc"); // 5139389 + addPowData("3ff005468a327822", POW, "3ff00000acc20750"); // 5151258 + addPowData("3ff00006666f30ff", POW, "3ff0000000d1b80e"); // 5185021 + addPowData("3ff000045a0b2035", POW, "3ff00000008e98e6"); // 5295829 + addPowData("3ff00e00380e10d7", POW, "3ff00001c9ff83c8"); // 5380897 + addPowData("3ff00c15de2b0d5e", POW, "3ff000018b6eaab6"); // 5400886 + addPowData("3ff00042afe6956a", POW, "3ff0000008892244"); // 5864127 + addPowData("3ff0005b7357c2d4", POW, "3ff000000bb48572"); // 6167339 + addPowData("3ff00033d5ab51c8", POW, "3ff0000006a279c8"); // 6240974 + addPowData("3ff0000046d74585", POW, "3ff0000000091150"); // 6279093 + addPowData("3ff0010403f34767", POW, "3ff0000021472146"); // 6428736 + addPowData("3ff00496fe59bc98", POW, "3ff000009650a4ca"); // 6432355,6493373 + addPowData("3ff0012e43815868", POW, "3ff0000026af266e"); // 6555029 + addPowData("3ff00021f6080e3c", POW, "3ff000000458d16a"); // 7092933 + addPowData("3ff000489c0f28bd", POW, "3ff00000094b3072"); // 7112412 + addPowData("3ff00009d3df2e9c", POW, "3ff00000014207b4"); // 7675535 + addPowData("3ff000def05fa9c8", POW, "3ff000001c887cdc"); // 7860324 + addPowData("3ff0013bca543227", POW, "3ff00000286a42d2"); // 8292427 + addPowData("3ff0021a2f14a0ee", POW, "3ff0000044deb040"); // 8517311 + addPowData("3ff0002cc166be3c", POW, "3ff0000005ba841e"); // 8763101 + addPowData("3ff0000cc84e613f", POW, "3ff0000001a2da46"); // 9269124 + addPowData("3ff000057b83c83f", POW, "3ff0000000b3a640"); // 9631452 + // add pow data + } + + private static void addPowData(String a, String b, String ret) { + powData.put(new PowData(hexToDouble(a), hexToDouble(b)), hexToDouble(ret)); + } + + private static double hexToDouble(String input) { + // Convert the hex string to a long + long hexAsLong = Long.parseLong(input, 16); + // and then convert the long to a double + return Double.longBitsToDouble(hexAsLong); + } + + private static class PowData { + final double a; + final double b; + + public PowData(double a, double b) { + this.a = a; + this.b = b; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + PowData powData = (PowData) o; + return Double.compare(powData.a, a) == 0 && Double.compare(powData.b, b) == 0; + } + + @Override + public int hashCode() { + return Objects.hash(a, b); + } + } + + /** + * *** methods are same as {@link java.lang.Math} methods, guaranteed by the call start *** + */ + + /** + * finally calls {@link java.lang.Math#addExact(long, long)} + */ + + public static long addExact(long x, long y) { + return StrictMath.addExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#addExact(int, int)} + */ + + public static int addExact(int x, int y) { + return StrictMath.addExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#subtractExact(long, long)} + */ + + public static long subtractExact(long x, long y) { + return StrictMath.subtractExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#floorMod(long, long)} + */ + public static long multiplyExact(long x, long y) { + return StrictMath.multiplyExact(x, y); + } + + public static long multiplyExact(long x, int y) { + return multiplyExact(x, (long) y); + } + + public static int multiplyExact(int x, int y) { + return StrictMath.multiplyExact(x, y); + } + + /** + * finally calls {@link java.lang.Math#floorDiv(long, long)} + */ + public static long floorDiv(long x, long y) { + return StrictMath.floorDiv(x, y); + } + + public static long floorDiv(long x, int y) { + return floorDiv(x, (long) y); + } + + /** + * finally calls {@link java.lang.Math#min(int, int)} + */ + public static int min(int a, int b) { + return StrictMath.min(a, b); + } + + /** + * finally calls {@link java.lang.Math#min(long, long)} + */ + public static long min(long a, long b) { + return StrictMath.min(a, b); + } + + /** + * finally calls {@link java.lang.Math#max(int, int)} + */ + public static int max(int a, int b) { + return StrictMath.max(a, b); + } + + /** + * finally calls {@link java.lang.Math#max(long, long)} + */ + public static long max(long a, long b) { + return StrictMath.max(a, b); + } + + /** + * finally calls {@link java.lang.Math#round(float)} + */ + public static int round(float a) { + return StrictMath.round(a); + } + + /** + * finally calls {@link java.lang.Math#round(double)} + */ + public static long round(double a) { + return StrictMath.round(a); + } + + /** + * finally calls {@link java.lang.Math#signum(double)} + */ + public static double signum(double d) { + return StrictMath.signum(d); + } + + /** + * finally calls {@link java.lang.Math#signum(float)} + */ + public static long abs(long a) { + return StrictMath.abs(a); + } + + /** + * *** methods are same as {@link java.lang.Math} methods, guaranteed by the call end *** + */ + + /** + * *** methods are same as {@link java.lang.Math} methods by mathematical algorithms*** + * / + + + /** + * mathematical integer: ceil(i) = floor(i) = i + * @return the smallest (closest to negative infinity) double value that is greater + * than or equal to the argument and is equal to a mathematical integer. + */ + public static double ceil(double a) { + return StrictMath.ceil(a); + } + + /** + * *** methods are no matters *** + */ + public static double random() { + return StrictMath.random(); + } + +} diff --git a/common/src/main/java/org/tron/common/math/MathWrapper.java b/platform/src/main/java/x86/org/tron/common/math/MathWrapper.java similarity index 100% rename from common/src/main/java/org/tron/common/math/MathWrapper.java rename to platform/src/main/java/x86/org/tron/common/math/MathWrapper.java From b30224ffc79ee058c3171a4720cb857ea7ef37a6 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Fri, 14 Mar 2025 20:49:13 +0800 Subject: [PATCH 11/16] feat(build): update verification-metadata and explicit_dependency --- framework/build.gradle | 2 +- gradle/verification-metadata.xml | 134 +++++++++++++++++++++---------- plugins/build.gradle | 2 +- 3 files changed, 92 insertions(+), 46 deletions(-) diff --git a/framework/build.gradle b/framework/build.gradle index dc7249a725e..7da69bc3dfc 100644 --- a/framework/build.gradle +++ b/framework/build.gradle @@ -157,7 +157,7 @@ def binaryRelease(taskName, jarName, mainClass) { // explicit_dependency dependsOn (project(':actuator').jar, project(':consensus').jar, project(':chainbase').jar, - project(':crypto').jar, project(':common').jar, project(':protocol').jar) + project(':crypto').jar, project(':common').jar, project(':protocol').jar, project(':platform').jar) from { configurations.runtimeClasspath.collect { diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 34ab41b748a..b57fef81536 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -547,9 +547,16 @@ + + + + + + + @@ -570,6 +577,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -791,35 +827,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -909,9 +916,16 @@ + + + + + + + @@ -1107,6 +1121,14 @@ + + + + + + + + @@ -1139,6 +1161,14 @@ + + + + + + + + @@ -1540,28 +1570,28 @@ - - - + + + - - + + - - - + + + - - + + - - - + + + - - + + @@ -2088,6 +2118,14 @@ + + + + + + + + @@ -2125,6 +2163,14 @@ + + + + + + + + diff --git a/plugins/build.gradle b/plugins/build.gradle index 7745ccf3f79..389812f65da 100644 --- a/plugins/build.gradle +++ b/plugins/build.gradle @@ -93,7 +93,7 @@ def binaryRelease(taskName, jarName, mainClass) { from(sourceSets.main.output) { include "/**" } - dependsOn project(':protocol').jar // explicit_dependency + dependsOn (project(':protocol').jar, project(':platform').jar) // explicit_dependency from { configurations.runtimeClasspath.collect { // https://docs.gradle.org/current/userguide/upgrading_version_6.html#changes_6.3 it.isDirectory() ? it : zipTree(it) From 465850c3f66e5a38bd2796917b1e5d5fca5e6563 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Sat, 15 Mar 2025 20:42:57 +0800 Subject: [PATCH 12/16] fix(test): NumberFormatException for JDK-8176425 --- .../test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java b/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java index 7af59f28ace..bda31656312 100644 --- a/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java +++ b/framework/src/test/java/org/tron/core/jsonrpc/JsonrpcServiceTest.java @@ -426,7 +426,8 @@ public void testGetByJsonBlockId() { try { getByJsonBlockId("0xxabc", wallet); } catch (Exception e) { - Assert.assertEquals("For input string: \"xabc\"", e.getMessage()); + // https://bugs.openjdk.org/browse/JDK-8176425, from JDK 12, the exception message is changed + Assert.assertTrue(e.getMessage().startsWith("For input string: \"xabc\"")); } } From 6ce0294f079461b53b8cec6bfeab157a126f842c Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Sat, 15 Mar 2025 20:59:15 +0800 Subject: [PATCH 13/16] feat(*): remove deprecated and marked for removal code --- .../java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java | 2 +- actuator/src/main/java/org/tron/core/vm/repository/Type.java | 2 +- actuator/src/main/java/org/tron/core/vm/repository/Value.java | 2 +- crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java | 2 +- crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java | 2 +- crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java | 2 +- .../org/tron/core/services/http/GetProposalByIdServlet.java | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java b/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java index 7997aaedcd5..b2f611cb24e 100644 --- a/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java +++ b/actuator/src/main/java/org/tron/core/vm/program/invoke/ProgramInvokeImpl.java @@ -314,7 +314,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(Boolean.valueOf(byTestingSuite).hashCode() + return Integer.valueOf(Boolean.valueOf(byTestingSuite).hashCode() + Boolean.valueOf(byTransaction).hashCode() + address.hashCode() + balance.hashCode() diff --git a/actuator/src/main/java/org/tron/core/vm/repository/Type.java b/actuator/src/main/java/org/tron/core/vm/repository/Type.java index e0842e6a593..b22e946f431 100644 --- a/actuator/src/main/java/org/tron/core/vm/repository/Type.java +++ b/actuator/src/main/java/org/tron/core/vm/repository/Type.java @@ -73,7 +73,7 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return new Integer(type).hashCode(); + return Integer.valueOf(type).hashCode(); } @Override diff --git a/actuator/src/main/java/org/tron/core/vm/repository/Value.java b/actuator/src/main/java/org/tron/core/vm/repository/Value.java index bf5d99c9c94..5b2be56be53 100644 --- a/actuator/src/main/java/org/tron/core/vm/repository/Value.java +++ b/actuator/src/main/java/org/tron/core/vm/repository/Value.java @@ -58,6 +58,6 @@ public boolean equals(Object obj) { @Override public int hashCode() { - return new Integer(type.hashCode() + Objects.hashCode(value)).hashCode(); + return Integer.valueOf(type.hashCode() + Objects.hashCode(value)).hashCode(); } } diff --git a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java index 26ea708fbe4..4fdb5da9712 100644 --- a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java +++ b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp12.java @@ -356,7 +356,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(a.hashCode() + b.hashCode()).hashCode(); + return Integer.valueOf(a.hashCode() + b.hashCode()).hashCode(); } @Override diff --git a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java index ba2a1ceb477..cecb045bbbb 100644 --- a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java +++ b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp2.java @@ -164,7 +164,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(a.hashCode() + b.hashCode()).hashCode(); + return Integer.valueOf(a.hashCode() + b.hashCode()).hashCode(); } Fp2 frobeniusMap(int power) { diff --git a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java index 0636cc334f1..6e57179f3e2 100644 --- a/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java +++ b/crypto/src/main/java/org/tron/common/crypto/zksnark/Fp6.java @@ -246,6 +246,6 @@ public boolean equals(Object o) { @Override public int hashCode() { - return new Integer(a.hashCode() + b.hashCode() + c.hashCode()).hashCode(); + return Integer.valueOf(a.hashCode() + b.hashCode() + c.hashCode()).hashCode(); } } diff --git a/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java b/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java index a903a5b4920..d13984217b9 100644 --- a/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java +++ b/framework/src/main/java/org/tron/core/services/http/GetProposalByIdServlet.java @@ -26,7 +26,7 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) { try { boolean visible = Util.getVisible(request); String input = request.getParameter("id"); - long id = new Long(input); + long id = Long.valueOf(input); fillResponse(ByteString.copyFrom(ByteArray.fromLong(id)), visible, response); } catch (Exception e) { Util.processError(e, response); From 36a4679f8b7c0cb842e926a40da77e4148536ca4 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Sat, 15 Mar 2025 21:33:34 +0800 Subject: [PATCH 14/16] feat(ci): add build for arm and x86 --- .github/workflows/arm64-platform.yml | 54 +++++++++++++++++++++++++++ .github/workflows/x86_64-platform.yml | 54 +++++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 .github/workflows/arm64-platform.yml create mode 100644 .github/workflows/x86_64-platform.yml diff --git a/.github/workflows/arm64-platform.yml b/.github/workflows/arm64-platform.yml new file mode 100644 index 00000000000..45d0cac8a98 --- /dev/null +++ b/.github/workflows/arm64-platform.yml @@ -0,0 +1,54 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle + +name: arm64-Platform Matrix + +on: + push: + branches: [ 'develop', 'master', 'release_**' ] + pull_request: + branches: [ 'develop', 'master', 'release_**' ] + +jobs: + build: + + name: ${{ matrix.os }} - ${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + + strategy: + fail-fast: false + matrix: + include: + # Apple Silicon runners (using latest available) + # see https://github.com/actions/runner-images?tab=readme-ov-file#available-images + - os: macOS + arch: arm64 + runner: macos-latest + # Linux arm runners + - os: Linux + arch: arm64 + runner: ubuntu-24.04-arm + + permissions: + contents: read + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + + # Configure Gradle for optimal use in GitHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Build with Gradle Wrapper + run: ./gradlew clean build --no-daemon diff --git a/.github/workflows/x86_64-platform.yml b/.github/workflows/x86_64-platform.yml new file mode 100644 index 00000000000..6bc9a9a9d1a --- /dev/null +++ b/.github/workflows/x86_64-platform.yml @@ -0,0 +1,54 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. +# This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle + +name: X86_64-Platform Matrix + +on: + push: + branches: [ 'develop', 'master', 'release_**' ] + pull_request: + branches: [ 'develop', 'master', 'release_**' ] + +jobs: + build: + + name: ${{ matrix.os }} - ${{ matrix.arch }} + runs-on: ${{ matrix.runner }} + + strategy: + fail-fast: false + matrix: + include: + # Macos Intel runners + # see https://github.com/actions/runner-images?tab=readme-ov-file#available-images + - os: macOS + arch: x86_64 + runner: macos-13 + # Linux x86_64 runners + - os: Linux + arch: x86_64 + runner: ubuntu-latest + + permissions: + contents: read + + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - name: Set up JDK 8 + uses: actions/setup-java@v4 + with: + java-version: '8' + distribution: 'zulu' + + # Configure Gradle for optimal use in GitHub Actions, including caching of downloaded dependencies. + # See: https://github.com/gradle/actions/blob/main/setup-gradle/README.md + - name: Setup Gradle + uses: gradle/actions/setup-gradle@v4 + + - name: Build with Gradle Wrapper + run: ./gradlew clean build --no-daemon From 3a6d3c134ef70bf51ef9504f6ed62f320bc0e469 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Sat, 15 Mar 2025 21:35:09 +0800 Subject: [PATCH 15/16] test(ci):fix test failed --- .../java/org/tron/core/jsonrpc/ApiUtilTest.java | 14 ++++++++++++++ .../src/test/java/org/tron/core/net/BaseNet.java | 11 +++++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java b/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java index 570e7ed3498..2c9afbac99b 100644 --- a/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java +++ b/framework/src/test/java/org/tron/core/jsonrpc/ApiUtilTest.java @@ -4,10 +4,13 @@ import static org.tron.keystore.Wallet.generateRandomBytes; import com.google.protobuf.ByteString; +import org.junit.AfterClass; import org.junit.Assert; +import org.junit.BeforeClass; import org.junit.Test; import org.tron.common.utils.ByteArray; import org.tron.core.capsule.BlockCapsule; +import org.tron.core.config.args.Args; import org.tron.core.services.jsonrpc.JsonRpcApiUtil; import org.tron.protos.Protocol.Block; import org.tron.protos.Protocol.BlockHeader; @@ -16,6 +19,17 @@ public class ApiUtilTest { + + @BeforeClass + public static void init() { + Args.setParam(new String[]{}, "config-localtest.conf"); + } + + @AfterClass + public static void clear() { + Args.clearParam(); + } + @Test public void testGetBlockID() { byte[] mockedHash = generateRandomBytes(128); diff --git a/framework/src/test/java/org/tron/core/net/BaseNet.java b/framework/src/test/java/org/tron/core/net/BaseNet.java index 65771bae952..fc04e210acd 100644 --- a/framework/src/test/java/org/tron/core/net/BaseNet.java +++ b/framework/src/test/java/org/tron/core/net/BaseNet.java @@ -16,6 +16,7 @@ import java.io.File; import java.io.IOException; import java.util.Collection; +import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -123,10 +124,12 @@ public static void init() throws Exception { @AfterClass public static void destroy() { - Collection peerConnections = ReflectUtils - .invokeMethod(tronNetDelegate, "getActivePeer"); - for (PeerConnection peer : peerConnections) { - peer.getChannel().close(); + if (Objects.nonNull(tronNetDelegate)) { + Collection peerConnections = ReflectUtils + .invokeMethod(tronNetDelegate, "getActivePeer"); + for (PeerConnection peer : peerConnections) { + peer.getChannel().close(); + } } Args.clearParam(); context.destroy(); From 6de651d971269935dd7c845837c422c9e0938314 Mon Sep 17 00:00:00 2001 From: halibobo1205 Date: Tue, 18 Mar 2025 15:53:53 +0800 Subject: [PATCH 16/16] feat(state): add state check --- .../main/java/org/tron/common/math/Maths.java | 70 +++++++- .../java/org/tron/core/ChainBaseManager.java | 5 + .../org/tron/core/capsule/BlockCapsule.java | 10 ++ .../java/org/tron/core/db/TronDatabase.java | 4 + .../tron/core/db2/core/SnapshotManager.java | 11 +- .../tron/core/service/RootHashService.java | 159 ++++++++++++++++++ .../tron/core/store/CheckPointV2Store.java | 41 ++++- .../org/tron/core/store/CheckTmpStore.java | 18 +- .../core/store/CorruptedCheckpointStore.java | 49 ++++++ .../java/org/tron/core/store/MathStore.java | 43 +++++ .../org/tron/core/store/StateRootStore.java | 54 ++++++ .../org/tron/core/store/StrictMathStore.java | 43 +++++ .../tron/common/context/GlobalContext.java | 47 ++++++ .../common/parameter/CommonParameter.java | 5 + .../org/tron/common/runtime/vm/DataWord.java | 9 +- .../java/org/tron/common/utils/ByteUtil.java | 6 +- .../java/org/tron/core/config/args/Args.java | 2 + .../main/java/org/tron/core/db/Manager.java | 17 +- .../org/tron/core/services/RpcApiService.java | 12 ++ .../tron/core/db2/SnapshotManagerTest.java | 32 ++-- .../tron/core/zksnark/LibrustzcashTest.java | 2 + protocol/src/main/protos/core/Tron.proto | 1 + 22 files changed, 599 insertions(+), 41 deletions(-) rename {common => chainbase}/src/main/java/org/tron/common/math/Maths.java (58%) create mode 100644 chainbase/src/main/java/org/tron/core/service/RootHashService.java create mode 100644 chainbase/src/main/java/org/tron/core/store/CorruptedCheckpointStore.java create mode 100644 chainbase/src/main/java/org/tron/core/store/MathStore.java create mode 100644 chainbase/src/main/java/org/tron/core/store/StateRootStore.java create mode 100644 chainbase/src/main/java/org/tron/core/store/StrictMathStore.java create mode 100644 common/src/main/java/org/tron/common/context/GlobalContext.java diff --git a/common/src/main/java/org/tron/common/math/Maths.java b/chainbase/src/main/java/org/tron/common/math/Maths.java similarity index 58% rename from common/src/main/java/org/tron/common/math/Maths.java rename to chainbase/src/main/java/org/tron/common/math/Maths.java index f0503cee2e4..38688d85014 100644 --- a/common/src/main/java/org/tron/common/math/Maths.java +++ b/chainbase/src/main/java/org/tron/common/math/Maths.java @@ -1,13 +1,45 @@ package org.tron.common.math; +import com.google.common.primitives.Bytes; +import java.nio.ByteBuffer; +import java.util.Optional; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.tron.common.context.GlobalContext; +import org.tron.core.store.MathStore; +import org.tron.core.store.StrictMathStore; + /** * This class is deprecated and should not be used in new code, * for cross-platform consistency, please use {@link StrictMathWrapper} instead, * especially for floating-point calculations. */ @Deprecated +@Component +@Slf4j(topic = "math") public class Maths { + private static Optional mathStore = Optional.empty(); + private static Optional strictMathStore = Optional.empty(); + + @Autowired + public Maths(@Autowired MathStore mathStore, @Autowired StrictMathStore strictMathStore) { + Maths.mathStore = Optional.ofNullable(mathStore); + Maths.strictMathStore = Optional.ofNullable(strictMathStore); + } + + private enum Op { + + POW((byte) 0x01); + + private final byte code; + + Op(byte code) { + this.code = code; + } + } + /** * Returns the value of the first argument raised to the power of the second argument. * @param a the base. @@ -15,7 +47,43 @@ public class Maths { * @return the value {@code a}{@code b}. */ public static double pow(double a, double b, boolean useStrictMath) { - return useStrictMath ? StrictMathWrapper.pow(a, b) : MathWrapper.pow(a, b); + double result = MathWrapper.pow(a, b); + double strictResult = StrictMathWrapper.pow(a, b); + if (useStrictMath) { + return strictResult; + } + final boolean isNoStrict = Double.compare(result, strictResult) != 0; + Optional header = GlobalContext.getHeader(); + header.ifPresent(h -> { + byte[] key = Bytes.concat(longToBytes(h), new byte[]{Op.POW.code}, + doubleToBytes(a), doubleToBytes(b)); + if (isNoStrict) { + logger.info("{}\t{}\t{}\t{}\t{}\t{}", h, Op.POW.code, doubleToHex(a), doubleToHex(b), + doubleToHex(result), doubleToHex(strictResult)); + } + mathStore.ifPresent(s -> s.put(key, doubleToBytes(result))); + strictMathStore.ifPresent(s -> s.put(key, doubleToBytes(strictResult))); + }); + return result; + } + + static String doubleToHex(double input) { + // Convert the starting value to the equivalent value in a long + long doubleAsLong = Double.doubleToRawLongBits(input); + // and then convert the long to a hex string + return Long.toHexString(doubleAsLong); + } + + private static byte[] doubleToBytes(double value) { + ByteBuffer buffer = ByteBuffer.allocate(Double.BYTES); + buffer.putDouble(value); + return buffer.array(); + } + + private static byte[] longToBytes(long value) { + ByteBuffer buffer = ByteBuffer.allocate(Long.BYTES); + buffer.putLong(value); + return buffer.array(); } /** diff --git a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java index 21f0bac8d77..0b19c1f7c9b 100644 --- a/chainbase/src/main/java/org/tron/core/ChainBaseManager.java +++ b/chainbase/src/main/java/org/tron/core/ChainBaseManager.java @@ -59,6 +59,7 @@ import org.tron.core.store.NullifierStore; import org.tron.core.store.ProposalStore; import org.tron.core.store.SectionBloomStore; +import org.tron.core.store.StateRootStore; import org.tron.core.store.StorageRowStore; import org.tron.core.store.TransactionHistoryStore; import org.tron.core.store.TransactionRetStore; @@ -233,6 +234,10 @@ public class ChainBaseManager { @Getter private SectionBloomStore sectionBloomStore; + @Autowired + @Getter + private StateRootStore stateRootStore; + @Autowired private DbStatService dbStatService; diff --git a/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java b/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java index 01ff7fb5365..a5053ccd2d6 100755 --- a/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java +++ b/chainbase/src/main/java/org/tron/core/capsule/BlockCapsule.java @@ -335,6 +335,16 @@ public String toString() { return toStringBuff.toString(); } + public Sha256Hash getStateRoot() { + ByteString stateRoot = this.block.getBlockHeader().getStateRoot(); + return stateRoot.isEmpty() ? Sha256Hash.ZERO_HASH : Sha256Hash.wrap(stateRoot); + } + + public void clearStateRoot() { + BlockHeader blockHeader = this.block.getBlockHeader().toBuilder().clearStateRoot().build(); + this.block = this.block.toBuilder().setBlockHeader(blockHeader).build(); + } + public static class BlockId extends Sha256Hash { private long num; diff --git a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java index 8630fbdcdff..30c3d1bfda7 100644 --- a/chainbase/src/main/java/org/tron/core/db/TronDatabase.java +++ b/chainbase/src/main/java/org/tron/core/db/TronDatabase.java @@ -70,6 +70,10 @@ public DbSourceInter getDbSource() { } public void updateByBatch(Map rows) { + this.updateByBatch(rows, writeOptions); + } + + public void updateByBatch(Map rows, WriteOptionsWrapper writeOptions) { this.dbSource.updateByBatch(rows, writeOptions); } diff --git a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java index eb27141a82c..42b9a1230e2 100644 --- a/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java +++ b/chainbase/src/main/java/org/tron/core/db2/core/SnapshotManager.java @@ -25,6 +25,7 @@ import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectFactory; import org.springframework.beans.factory.annotation.Autowired; import org.tron.common.error.TronDBException; import org.tron.common.es.ExecutorServiceManager; @@ -80,6 +81,9 @@ public class SnapshotManager implements RevokingDatabase { @Getter private CheckTmpStore checkTmpStore; + @Autowired + private ObjectFactory checkPointV2Store; + @Setter private volatile int maxFlushCount = DEFAULT_MIN_FLUSH_COUNT; @@ -387,15 +391,14 @@ public void createCheckpoint() { } } if (isV2Open()) { - String dbName = String.valueOf(System.currentTimeMillis()); - checkPointStore = getCheckpointDB(dbName); + checkPointStore = checkPointV2Store.getObject(); syncFlag = CommonParameter.getInstance().getStorage().isCheckpointSync(); } else { checkPointStore = checkTmpStore; syncFlag = CommonParameter.getInstance().getStorage().isDbSync(); } - checkPointStore.getDbSource().updateByBatch(batch.entrySet().stream() + checkPointStore.updateByBatch(batch.entrySet().stream() .map(e -> Maps.immutableEntry(e.getKey().getBytes(), e.getValue().getBytes())) .collect(HashMap::new, (m, k) -> m.put(k.getKey(), k.getValue()), HashMap::putAll), WriteOptionsWrapper.getInstance().sync(syncFlag)); @@ -410,7 +413,7 @@ public void createCheckpoint() { } private TronDatabase getCheckpointDB(String dbName) { - return new CheckPointV2Store(CHECKPOINT_V2_DIR+"/"+dbName); + return new CheckPointV2Store(CHECKPOINT_V2_DIR, dbName); } public List getCheckpointList() { diff --git a/chainbase/src/main/java/org/tron/core/service/RootHashService.java b/chainbase/src/main/java/org/tron/core/service/RootHashService.java new file mode 100644 index 00000000000..c83c028e1ac --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/service/RootHashService.java @@ -0,0 +1,159 @@ +package org.tron.core.service; + +import com.google.common.collect.Sets; +import com.google.common.collect.Streams; +import com.google.common.primitives.Bytes; +import com.google.common.primitives.Ints; +import com.google.protobuf.ByteString; +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.tron.common.context.GlobalContext; +import org.tron.common.error.TronDBException; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.MerkleRoot; +import org.tron.common.utils.Pair; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.db.TronDatabase; +import org.tron.core.db2.common.Value; +import org.tron.core.store.AccountAssetStore; +import org.tron.core.store.CorruptedCheckpointStore; +import org.tron.protos.Protocol; + +@Slf4j(topic = "DB") +@Component +public class RootHashService { + + private static final byte[] HEADER_KEY = "latest_block_header_number".getBytes(); + + private static Optional corruptedCheckpointStore = Optional.empty(); + private static AccountAssetStore assetStore; + private static final List stateDbs = Arrays.asList( + "account", "account-asset", "asset-issue-v2", + "code", "contract", "contract-state", "storage-row", + "delegation", "DelegatedResource", + "exchange-v2", + "market_account", "market_order", "market_pair_price_to_order", "market_pair_to_price", + "properties", "proposal", + "votes", "witness", "witness_schedule" + ); + private static final byte[] CURRENT_SHUFFLED_WITNESSES = "current_shuffled_witnesses".getBytes(); + private static final String FORK_PREFIX = "FORK_VERSION_"; + private static final String DONE_SUFFIX = "_DONE"; + private static final String ACCOUNT_VOTE_SUFFIX = "-account-vote"; + private static final Set ignoredProperties = Sets.newHashSet( + "VOTE_REWARD_RATE", "SINGLE_REPEAT", "NON_EXISTENT_ACCOUNT_TRANSFER_MIN", + "ALLOW_TVM_ASSET_ISSUE", "ALLOW_TVM_STAKE", + "MAX_VOTE_NUMBER", "MAX_FROZEN_NUMBER", "MAINTENANCE_TIME_INTERVAL", + "LATEST_SOLIDIFIED_BLOCK_NUM", "BLOCK_NET_USAGE", + "BLOCK_FILLED_SLOTS_INDEX", "BLOCK_FILLED_SLOTS_NUMBER", "BLOCK_FILLED_SLOTS"); + + @Autowired + public RootHashService(@Autowired CorruptedCheckpointStore corruptedCheckpointStore, + @Autowired AccountAssetStore assetStore) { + RootHashService.corruptedCheckpointStore = Optional.ofNullable(corruptedCheckpointStore); + RootHashService.assetStore = assetStore; + } + + public static Pair, Sha256Hash> getRootHash(Map rows) { + try { + Map preparedStateData = preparedStateData(rows); + AtomicReference> height = new AtomicReference<>(Optional.empty()); + List ids = Streams.stream(preparedStateData.entrySet()).parallel().map(entry -> { + if (Arrays.equals(HEADER_KEY, entry.getKey())) { + height.set(Optional.of(ByteArray.toLong(entry.getValue()))); + } + return getHash(entry); + }).sorted().collect(Collectors.toList()); + Sha256Hash actual = MerkleRoot.root(ids); + long num = height.get().orElseThrow(() -> new TronDBException("blockNum is null")); + Optional expected = GlobalContext.popBlockHash(num); + if (expected.isPresent() && !Objects.equals(expected.get(), actual)) { + corruptedCheckpointStore.ifPresent(TronDatabase::reset); + corruptedCheckpointStore.ifPresent(store -> store.updateByBatch(rows)); + throw new TronDBException(String.format( + "Root hash mismatch for blockNum: %s, expected: %s, actual: %s", + num, expected, actual)); + } + return new Pair<>(height.get(), actual); + } catch (IOException e) { + throw new TronDBException(e); + } + } + + private static Map preparedStateData(Map rows) + throws IOException { + Map preparedStateData = new HashMap<>(rows.size()); + for (Map.Entry e : rows.entrySet()) { + byte[] key = e.getKey(); + String dbName = simpleDecode(key); + if (!stateDbs.contains(dbName)) { + continue; + } + byte[] realKey = Arrays.copyOfRange(key, dbName.getBytes().length + Integer.BYTES, + key.length); + if ("witness_schedule".equals(dbName) && Arrays.equals(realKey, CURRENT_SHUFFLED_WITNESSES)) { + continue; + } + if ("properties".equals(dbName)) { + String keyStr = new String(realKey); + if (ignoredProperties.contains(keyStr) + || keyStr.startsWith(FORK_PREFIX) || keyStr.endsWith(DONE_SUFFIX)) { + continue; + } + } + byte[] value = e.getValue(); + byte[] realValue = value.length == 1 ? null : Arrays.copyOfRange(value, 1, value.length); + if (realValue != null) { + if ("witness".equals(dbName)) { + realValue = Protocol.Witness.parseFrom(realValue) + .toBuilder().clearTotalMissed() + .build().toByteArray(); // ignore totalMissed + } + if ("account".equals(dbName)) { + Protocol.Account account = Protocol.Account.parseFrom(realValue); + Map assets = new TreeMap<>(assetStore.getAllAssets(account)); + assets.entrySet().removeIf(entry -> entry.getValue() == 0); + realValue = account.toBuilder().clearAsset().clearAssetV2().clearAssetOptimized() + .putAllAssetV2(assets) + .build().toByteArray(); + } + if ("delegation".equals(dbName) && new String(key).endsWith(ACCOUNT_VOTE_SUFFIX)) { + Protocol.Account account = Protocol.Account.parseFrom(realValue); + realValue = Protocol.Account.newBuilder().addAllVotes(account.getVotesList()) + .build().toByteArray(); + } + } + if (realValue != null) { + preparedStateData.put(realKey, realValue); + } else { + if (Value.Operator.DELETE.getValue() != value[0]) { + preparedStateData.put(realKey, ByteString.EMPTY.toByteArray()); + } + } + } + return preparedStateData; + } + + private static String simpleDecode(byte[] bytes) { + byte[] lengthBytes = Arrays.copyOf(bytes, Integer.BYTES); + int length = Ints.fromByteArray(lengthBytes); + byte[] value = Arrays.copyOfRange(bytes, Integer.BYTES, Integer.BYTES + length); + return new String(value); + } + + private static Sha256Hash getHash(Map.Entry entry) { + return Sha256Hash.of(true, Bytes.concat(entry.getKey(), entry.getValue())); + } +} diff --git a/chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java b/chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java index 2f952e6b82a..ae1199bc5a3 100644 --- a/chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java +++ b/chainbase/src/main/java/org/tron/core/store/CheckPointV2Store.java @@ -1,21 +1,39 @@ package org.tron.core.store; import com.google.protobuf.InvalidProtocolBufferException; +import java.util.Map; +import java.util.Optional; +import java.util.Spliterator; +import java.util.function.Consumer; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.beans.factory.config.ConfigurableBeanFactory; +import org.springframework.context.annotation.Scope; +import org.springframework.stereotype.Component; +import org.tron.common.storage.WriteOptionsWrapper; +import org.tron.common.utils.Pair; +import org.tron.common.utils.Sha256Hash; import org.tron.core.db.TronDatabase; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; - -import java.util.Spliterator; -import java.util.function.Consumer; +import org.tron.core.service.RootHashService; @Slf4j(topic = "DB") +@Component +@Scope(value = ConfigurableBeanFactory.SCOPE_PROTOTYPE/*, proxyMode = ScopedProxyMode.TARGET_CLASS*/) public class CheckPointV2Store extends TronDatabase { @Autowired - public CheckPointV2Store(String dbPath) { - super(dbPath); + private StateRootStore stateRootStore; + + @Autowired + public CheckPointV2Store(@Value("checkpoint") String dbPath) { + this(dbPath, String.valueOf(System.currentTimeMillis())); + } + + public CheckPointV2Store(String dbPath, String name) { + super(dbPath + "/" + name); } @Override @@ -57,14 +75,21 @@ protected void init() { */ @Override public void close() { - logger.debug("******** Begin to close {}. ********", getName()); + logger.debug("******** Begin to close {}. ********", getDbName()); try { dbSource.closeDB(); } catch (Exception e) { - logger.warn("Failed to close {}.", getName(), e); + logger.warn("Failed to close {}.", getDbName(), e); } finally { - logger.debug("******** End to close {}. ********", getName()); + logger.debug("******** End to close {}. ********", getDbName()); } } + @Override + public void updateByBatch(Map rows, WriteOptionsWrapper writeOptions) { + Pair, Sha256Hash> ret = RootHashService.getRootHash(rows); + super.updateByBatch(rows, writeOptions); + ret.getKey().ifPresent(height -> stateRootStore.put(height, ret.getValue())); + } + } diff --git a/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java b/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java index 09f60c83898..bceab7c4630 100644 --- a/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java +++ b/chainbase/src/main/java/org/tron/core/store/CheckTmpStore.java @@ -1,18 +1,27 @@ package org.tron.core.store; import com.google.protobuf.InvalidProtocolBufferException; +import java.util.Map; +import java.util.Optional; import java.util.Spliterator; import java.util.function.Consumer; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; import org.springframework.stereotype.Component; +import org.tron.common.storage.WriteOptionsWrapper; +import org.tron.common.utils.Pair; +import org.tron.common.utils.Sha256Hash; import org.tron.core.db.TronDatabase; import org.tron.core.exception.BadItemException; import org.tron.core.exception.ItemNotFoundException; +import org.tron.core.service.RootHashService; @Component public class CheckTmpStore extends TronDatabase { + @Autowired + private StateRootStore stateRootStore; + @Autowired public CheckTmpStore(ApplicationContext ctx) { super("tmp"); @@ -46,4 +55,11 @@ public void forEach(Consumer action) { public Spliterator spliterator() { return null; } -} \ No newline at end of file + + @Override + public void updateByBatch(Map rows, WriteOptionsWrapper writeOptions) { + Pair, Sha256Hash> ret = RootHashService.getRootHash(rows); + super.updateByBatch(rows, writeOptions); + ret.getKey().ifPresent(height -> stateRootStore.put(height, ret.getValue())); + } +} diff --git a/chainbase/src/main/java/org/tron/core/store/CorruptedCheckpointStore.java b/chainbase/src/main/java/org/tron/core/store/CorruptedCheckpointStore.java new file mode 100644 index 00000000000..efab9345871 --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/store/CorruptedCheckpointStore.java @@ -0,0 +1,49 @@ +package org.tron.core.store; + +import com.google.protobuf.InvalidProtocolBufferException; +import java.util.Spliterator; +import java.util.function.Consumer; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.tron.core.db.TronDatabase; +import org.tron.core.exception.BadItemException; +import org.tron.core.exception.ItemNotFoundException; + +@Component +public class CorruptedCheckpointStore extends TronDatabase { + + @Autowired + public CorruptedCheckpointStore(@Value("corrupted-checkpoint") String dbName) { + super(dbName); + } + + @Override + public void put(byte[] key, byte[] item) { + } + + @Override + public void delete(byte[] key) { + } + + @Override + public byte[] get(byte[] key) + throws InvalidProtocolBufferException, ItemNotFoundException, BadItemException { + return null; + } + + @Override + public boolean has(byte[] key) { + return false; + } + + @Override + public void forEach(Consumer action) { + + } + + @Override + public Spliterator spliterator() { + return null; + } +} diff --git a/chainbase/src/main/java/org/tron/core/store/MathStore.java b/chainbase/src/main/java/org/tron/core/store/MathStore.java new file mode 100644 index 00000000000..70266a5b67a --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/store/MathStore.java @@ -0,0 +1,43 @@ +package org.tron.core.store; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.tron.core.db.TronDatabase; +import org.tron.core.db.common.iterator.DBIterator; + +@Slf4j(topic = "DB") +@Component +public class MathStore extends TronDatabase { + + @Autowired + private MathStore(@Value("math") String dbName) { + super(dbName); + } + + @Override + public byte[] get(byte[] key) { + return dbSource.getData(key); + } + + @Override + public void put(byte[] key, byte[] item) { + dbSource.putData(key, item); + } + + @Override + public void delete(byte[] key) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean has(byte[] key) { + return dbSource.getData(key) != null; + } + + @Override + public DBIterator iterator() { + return ((DBIterator) dbSource.iterator()); + } +} diff --git a/chainbase/src/main/java/org/tron/core/store/StateRootStore.java b/chainbase/src/main/java/org/tron/core/store/StateRootStore.java new file mode 100644 index 00000000000..8a5cfbe586d --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/store/StateRootStore.java @@ -0,0 +1,54 @@ +package org.tron.core.store; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.tron.common.utils.ByteArray; +import org.tron.common.utils.Sha256Hash; +import org.tron.core.db.TronDatabase; +import org.tron.core.db.common.iterator.DBIterator; + +@Slf4j(topic = "DB") +@Component +public class StateRootStore extends TronDatabase { + + @Autowired + private StateRootStore(@Value("state-root") String dbName) { + super(dbName); + } + + @Override + public byte[] get(byte[] key) { + return dbSource.getData(key); + } + + public byte[] get(long key) { + return dbSource.getData(ByteArray.fromLong(key)); + } + + @Override + public void put(byte[] key, byte[] item) { + dbSource.putData(key, item); + } + + public void put(long key, Sha256Hash root) { + logger.info("block: {}, stateRoot: {}", key, root); + this.put(ByteArray.fromLong(key), root.getBytes()); + } + + @Override + public void delete(byte[] key) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean has(byte[] key) { + return dbSource.getData(key) != null; + } + + @Override + public DBIterator iterator() { + return ((DBIterator) dbSource.iterator()); + } +} diff --git a/chainbase/src/main/java/org/tron/core/store/StrictMathStore.java b/chainbase/src/main/java/org/tron/core/store/StrictMathStore.java new file mode 100644 index 00000000000..91303db3651 --- /dev/null +++ b/chainbase/src/main/java/org/tron/core/store/StrictMathStore.java @@ -0,0 +1,43 @@ +package org.tron.core.store; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.tron.core.db.TronDatabase; +import org.tron.core.db.common.iterator.DBIterator; + +@Slf4j(topic = "DB") +@Component +public class StrictMathStore extends TronDatabase { + + @Autowired + private StrictMathStore(@Value("strict-math") String dbName) { + super(dbName); + } + + @Override + public byte[] get(byte[] key) { + return dbSource.getData(key); + } + + @Override + public void put(byte[] key, byte[] item) { + dbSource.putData(key, item); + } + + @Override + public void delete(byte[] key) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean has(byte[] key) { + return dbSource.getData(key) != null; + } + + @Override + public DBIterator iterator() { + return ((DBIterator) dbSource.iterator()); + } +} diff --git a/common/src/main/java/org/tron/common/context/GlobalContext.java b/common/src/main/java/org/tron/common/context/GlobalContext.java new file mode 100644 index 00000000000..91bc262ea46 --- /dev/null +++ b/common/src/main/java/org/tron/common/context/GlobalContext.java @@ -0,0 +1,47 @@ +package org.tron.common.context; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import org.tron.common.utils.Sha256Hash; + +public class GlobalContext { + + private static final ThreadLocal HEADER = new ThreadLocal<>(); + private static final ThreadLocal> BLOCK_HASHES = + ThreadLocal.withInitial(() -> Collections.synchronizedMap(new HashMap<>())); + + private GlobalContext() { + } + + public static Optional getHeader() { + return Optional.ofNullable(HEADER.get()); + } + + public static void setHeader(long header) { + HEADER.set(header); + } + + public static void removeHeader() { + HEADER.remove(); + } + + public static void putBlockHash(long blockNumber, Sha256Hash blockHash) { + BLOCK_HASHES.get().put(blockNumber, blockHash); + } + + public static Optional popBlockHash(long blockNumber) { + return Optional.ofNullable(BLOCK_HASHES.get().remove(blockNumber)); + } + + public static void clearBlockHashes() { + BLOCK_HASHES.get().clear(); + } + + public static void clear() { + removeHeader(); + clearBlockHashes(); + } + +} diff --git a/common/src/main/java/org/tron/common/parameter/CommonParameter.java b/common/src/main/java/org/tron/common/parameter/CommonParameter.java index 16fb752d82a..0942bad5215 100644 --- a/common/src/main/java/org/tron/common/parameter/CommonParameter.java +++ b/common/src/main/java/org/tron/common/parameter/CommonParameter.java @@ -717,6 +717,11 @@ public class CommonParameter { @Setter public long allowTvmBlob; + @Getter + @Setter + @Parameter(names = {"--check.root.disable"}, description = "disable check state root") + public boolean checkRootHashDisable = false; + private static double calcMaxTimeRatio() { //return max(2.0, min(5.0, 5 * 4.0 / max(Runtime.getRuntime().availableProcessors(), 1))); return 5.0; diff --git a/common/src/main/java/org/tron/common/runtime/vm/DataWord.java b/common/src/main/java/org/tron/common/runtime/vm/DataWord.java index faeae45782e..b078febb0e6 100644 --- a/common/src/main/java/org/tron/common/runtime/vm/DataWord.java +++ b/common/src/main/java/org/tron/common/runtime/vm/DataWord.java @@ -17,8 +17,8 @@ */ package org.tron.common.runtime.vm; -import static org.tron.common.math.Maths.min; -import static org.tron.common.math.Maths.signum; +import static org.tron.common.math.StrictMathWrapper.min; +import static org.tron.common.math.StrictMathWrapper.signum; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; @@ -31,7 +31,6 @@ import org.tron.common.utils.DecodeUtil; import org.tron.common.utils.FastByteComparisons; import org.tron.core.db.ByteArrayWrapper; -import org.tron.core.vm.config.VMConfig; /** * DataWord is the 32-byte array representation of a 256-bit number Calculations can be done on this @@ -169,7 +168,7 @@ public byte[] getClonedData() { byte[] ret = ByteUtil.EMPTY_BYTE_ARRAY; if (data != null) { ret = new byte[WORD_SIZE]; - int dataSize = min(data.length, WORD_SIZE, VMConfig.disableJavaLangMath()); + int dataSize = min(data.length, WORD_SIZE); System.arraycopy(data, 0, ret, 0, dataSize); } return ret; @@ -488,7 +487,7 @@ public int compareTo(DataWord o) { data, 0, data.length, o.getData(), 0, o.getData().length); // Convert result into -1, 0 or 1 as is the convention - return (int) signum(result, VMConfig.disableJavaLangMath()); + return (int) signum(result); } public void signExtend(byte k) { diff --git a/common/src/main/java/org/tron/common/utils/ByteUtil.java b/common/src/main/java/org/tron/common/utils/ByteUtil.java index 88d4cfdf90b..5f196ecb8ff 100644 --- a/common/src/main/java/org/tron/common/utils/ByteUtil.java +++ b/common/src/main/java/org/tron/common/utils/ByteUtil.java @@ -18,7 +18,7 @@ package org.tron.common.utils; -import static org.tron.common.math.Maths.min; +import static org.tron.common.math.StrictMathWrapper.min; import com.google.common.base.Preconditions; import com.google.common.primitives.UnsignedBytes; @@ -72,7 +72,7 @@ public static byte[] bigIntegerToBytes(BigInteger b, int numBytes) { byte[] bytes = new byte[numBytes]; byte[] biBytes = b.toByteArray(); int start = (biBytes.length == numBytes + 1) ? 1 : 0; - int length = min(biBytes.length, numBytes, true); + int length = min(biBytes.length, numBytes); System.arraycopy(biBytes, start, bytes, numBytes - length, length); return bytes; } @@ -348,7 +348,7 @@ public static byte[] parseBytes(byte[] input, int offset, int len) { } byte[] bytes = new byte[len]; - System.arraycopy(input, offset, bytes, 0, min(input.length - offset, len, true)); + System.arraycopy(input, offset, bytes, 0, min(input.length - offset, len)); return bytes; } diff --git a/framework/src/main/java/org/tron/core/config/args/Args.java b/framework/src/main/java/org/tron/core/config/args/Args.java index 64dffa8843e..92960c5fc54 100644 --- a/framework/src/main/java/org/tron/core/config/args/Args.java +++ b/framework/src/main/java/org/tron/core/config/args/Args.java @@ -50,6 +50,7 @@ import org.tron.common.args.GenesisBlock; import org.tron.common.args.Witness; import org.tron.common.config.DbBackupConfig; +import org.tron.common.context.GlobalContext; import org.tron.common.crypto.SignInterface; import org.tron.common.logsfilter.EventPluginConfig; import org.tron.common.logsfilter.FilterQuery; @@ -246,6 +247,7 @@ public static void clearParam() { PARAMETER.consensusLogicOptimization = 0; PARAMETER.allowTvmCancun = 0; PARAMETER.allowTvmBlob = 0; + GlobalContext.removeHeader(); } /** diff --git a/framework/src/main/java/org/tron/core/db/Manager.java b/framework/src/main/java/org/tron/core/db/Manager.java index 3635f38d9ad..3773300ca60 100644 --- a/framework/src/main/java/org/tron/core/db/Manager.java +++ b/framework/src/main/java/org/tron/core/db/Manager.java @@ -52,6 +52,7 @@ import org.tron.api.GrpcAPI.TransactionInfoList; import org.tron.common.args.GenesisBlock; import org.tron.common.bloom.Bloom; +import org.tron.common.context.GlobalContext; import org.tron.common.es.ExecutorServiceManager; import org.tron.common.exit.ExitManager; import org.tron.common.logsfilter.EventPluginLoader; @@ -1033,8 +1034,20 @@ public void pushVerifiedBlock(BlockCapsule block) throws ContractValidateExcepti NonCommonBlockException, BadNumberBlockException, BadBlockException, ZksnarkException, EventBloomException { block.generatedByMyself = true; - long start = System.currentTimeMillis(); - pushBlock(block); + final long start = System.currentTimeMillis(); + Sha256Hash stateRoot = block.getStateRoot(); + if (!CommonParameter.getInstance().isCheckRootHashDisable() + && !Objects.equals(Sha256Hash.ZERO_HASH, stateRoot)) { + GlobalContext.putBlockHash(block.getNum(), stateRoot); + } + // clear stateRoot for block + block.clearStateRoot(); + try { + GlobalContext.setHeader(block.getNum()); + pushBlock(block); + } finally { + GlobalContext.removeHeader(); + } logger.info("Push block cost: {} ms, blockNum: {}, blockHash: {}, trx count: {}.", System.currentTimeMillis() - start, block.getNum(), diff --git a/framework/src/main/java/org/tron/core/services/RpcApiService.java b/framework/src/main/java/org/tron/core/services/RpcApiService.java index 8f9c6b15bb7..8c9e081fe68 100755 --- a/framework/src/main/java/org/tron/core/services/RpcApiService.java +++ b/framework/src/main/java/org/tron/core/services/RpcApiService.java @@ -330,6 +330,7 @@ public void getNowBlock(EmptyMessage request, StreamObserver responseObse Block block = null; try { block = chainBaseManager.getHead().getInstance(); + block = addStateRoot(block); } catch (StoreException e) { logger.error(e.getMessage()); } @@ -342,6 +343,7 @@ public void getBlockByNum(NumberMessage request, StreamObserver responseO Block block = null; try { block = chainBaseManager.getBlockByNum(request.getNum()).getInstance(); + block = addStateRoot(block); } catch (StoreException e) { logger.error(e.getMessage()); } @@ -359,6 +361,16 @@ public void getDynamicProperties(EmptyMessage request, responseObserver.onNext(dynamicProperties); responseObserver.onCompleted(); } + + private Block addStateRoot(Block block) { + byte[] stateRoot = chainBaseManager.getStateRootStore().get( + block.getBlockHeader().getRawData().getNumber()); + if (stateRoot != null) { + block = block.toBuilder().setBlockHeader(block.getBlockHeader().toBuilder() + .setStateRoot(ByteString.copyFrom(stateRoot))).build(); + } + return block; + } } /** diff --git a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java index d6fd319e6a0..912011b2656 100644 --- a/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java +++ b/framework/src/test/java/org/tron/core/db2/SnapshotManagerTest.java @@ -18,8 +18,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; -import org.tron.common.application.Application; -import org.tron.common.application.ApplicationFactory; +import org.junit.rules.TestName; import org.tron.common.application.TronApplicationContext; import org.tron.common.utils.Sha256Hash; import org.tron.core.Constant; @@ -30,8 +29,6 @@ import org.tron.core.db2.SnapshotRootTest.ProtoCapsuleTest; import org.tron.core.db2.core.Chainbase; import org.tron.core.db2.core.SnapshotManager; -import org.tron.core.exception.BadItemException; -import org.tron.core.exception.ItemNotFoundException; import org.tron.core.exception.TronError; @Slf4j @@ -39,39 +36,37 @@ public class SnapshotManagerTest { private SnapshotManager revokingDatabase; private TronApplicationContext context; - private Application appT; private TestRevokingTronStore tronDatabase; @Rule - public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - + public final TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Rule + public TestName name = new TestName(); @Before public void init() throws IOException { Args.setParam(new String[]{"-d", temporaryFolder.newFolder().toString()}, Constant.TEST_CONF); context = new TronApplicationContext(DefaultConfig.class); - appT = ApplicationFactory.create(context); revokingDatabase = context.getBean(SnapshotManager.class); revokingDatabase.enable(); - tronDatabase = new TestRevokingTronStore("testSnapshotManager-test"); - revokingDatabase.add(tronDatabase.getRevokingDB()); } @After public void removeDb() { - Args.clearParam(); - context.destroy(); tronDatabase.close(); + Args.clearParam(); + context.close(); } @Test - public synchronized void testRefresh() - throws BadItemException, ItemNotFoundException { + public synchronized void testRefresh() { + tronDatabase = new TestRevokingTronStore(name.getMethodName()); + revokingDatabase.add(tronDatabase.getRevokingDB()); while (revokingDatabase.size() != 0) { revokingDatabase.pop(); } - revokingDatabase.setMaxFlushCount(0); + revokingDatabase.setMaxFlushCount(1); revokingDatabase.setUnChecked(false); revokingDatabase.setMaxSize(5); List dbList = revokingDatabase.getDbs(); @@ -79,6 +74,7 @@ public synchronized void testRefresh() .map(db -> Maps.immutableEntry(db.getDbName(), db)) .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); ProtoCapsuleTest protoCapsule = new ProtoCapsuleTest("refresh".getBytes()); + dbMap.get("properties").put("latest_block_header_number".getBytes(), Longs.toByteArray(0)); for (int i = 1; i < 11; i++) { ProtoCapsuleTest testProtoCapsule = new ProtoCapsuleTest(("refresh" + i).getBytes()); try (ISession tmpSession = revokingDatabase.buildSession()) { @@ -86,6 +82,7 @@ public synchronized void testRefresh() BlockCapsule blockCapsule = new BlockCapsule(i, Sha256Hash.ZERO_HASH, System.currentTimeMillis(), ByteString.EMPTY); dbMap.get("block").put(Longs.toByteArray(i), blockCapsule.getData()); + dbMap.get("properties").put("latest_block_header_number".getBytes(), Longs.toByteArray(i)); tmpSession.commit(); } } @@ -97,6 +94,8 @@ public synchronized void testRefresh() @Test public synchronized void testClose() { + tronDatabase = new TestRevokingTronStore(name.getMethodName()); + revokingDatabase.add(tronDatabase.getRevokingDB()); while (revokingDatabase.size() != 0) { revokingDatabase.pop(); } @@ -111,8 +110,7 @@ public synchronized void testClose() { tronDatabase.put(protoCapsule.getData(), testProtoCapsule); } } - Assert.assertEquals(null, - tronDatabase.get(protoCapsule.getData())); + Assert.assertNull(tronDatabase.get(protoCapsule.getData())); } diff --git a/framework/src/test/java/org/tron/core/zksnark/LibrustzcashTest.java b/framework/src/test/java/org/tron/core/zksnark/LibrustzcashTest.java index 049fb2528b1..c1304cbec29 100644 --- a/framework/src/test/java/org/tron/core/zksnark/LibrustzcashTest.java +++ b/framework/src/test/java/org/tron/core/zksnark/LibrustzcashTest.java @@ -29,6 +29,7 @@ import org.junit.Ignore; import org.junit.Test; import org.tron.common.BaseTest; +import org.tron.common.context.GlobalContext; import org.tron.common.utils.ByteArray; import org.tron.common.utils.ByteUtil; import org.tron.common.zksnark.IncrementalMerkleTreeContainer; @@ -85,6 +86,7 @@ public static void init() { "config-test-mainnet.conf" ); Args.setFullNodeAllowShieldedTransaction(true); + GlobalContext.setHeader(1); } private static int randomInt(int minInt, int maxInt) { diff --git a/protocol/src/main/protos/core/Tron.proto b/protocol/src/main/protos/core/Tron.proto index 2ffefbf9f3e..e9242c9e794 100644 --- a/protocol/src/main/protos/core/Tron.proto +++ b/protocol/src/main/protos/core/Tron.proto @@ -515,6 +515,7 @@ message BlockHeader { } raw raw_data = 1; bytes witness_signature = 2; + bytes state_root = 3; } // block