diff --git a/tools/gradlew/build.gradle b/tools/gradlew/build.gradle index 6029ef7e..60970845 100644 --- a/tools/gradlew/build.gradle +++ b/tools/gradlew/build.gradle @@ -1,7 +1,49 @@ +import org.gradle.nativeplatform.platform.internal.Architectures +import org.gradle.internal.os.OperatingSystem + plugins { id 'java' } +def arch = System.getProperty("os.arch").toLowerCase() +def javaVersion = JavaVersion.current() +def isArm64 = Architectures.AARCH64.isAlias(arch) +def archSource = isArm64 ? "arm" : "x86" +def isMac = OperatingSystem.current().isMacOsX() + +ext.archInfo = [ + name : arch, + java : javaVersion, + isArm64 : isArm64, + sourceSets: [ + main: [ + java: [ + srcDirs: ["src/main/java/common", "src/main/java/${archSource}"] + ] + ], + test: [ + java: [ + srcDirs: ["src/test/java"] + ] + ] + ], + requires: [ + JavaVersion: isArm64 ? JavaVersion.VERSION_17 : JavaVersion.VERSION_1_8, + RocksdbVersion: isArm64 ? '9.7.4' : '5.15.10', + // https://github.com/grpc/grpc-java/issues/7690 + // https://github.com/grpc/grpc-java/pull/12319, Add support for macOS aarch64 with universal binary + // https://github.com/grpc/grpc-java/pull/11371 , 1.64.x is not supported CentOS 7. + ProtocGenVersion: isArm64 && isMac ? '1.76.0' : '1.60.0' + ], + VMOptions: isArm64 ? "${rootDir}/gradle/jdk17/java-tron.vmoptions" : "${rootDir}/gradle/java-tron.vmoptions" +] + +if (!archInfo.java.is(archInfo.requires.JavaVersion)) { + throw new GradleException("Java ${archInfo.requires.JavaVersion} is required for ${archInfo.name}. Detected version ${archInfo.java}") +} + +println "Building for architecture: ${archInfo.name}, Java version: ${archInfo.java}" + tasks.named('jar') { enabled = false } diff --git a/tools/stress_test/build.gradle b/tools/stress_test/build.gradle index 894c81a8..0c8654a2 100644 --- a/tools/stress_test/build.gradle +++ b/tools/stress_test/build.gradle @@ -7,7 +7,7 @@ plugins { } sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' repositories { @@ -44,18 +44,25 @@ dependencies { implementation "io.netty:netty-all:4.1.100.Final" implementation("io.github.317787106:trident:0.9.2.5") implementation group: 'com.typesafe', name: 'config', version: '1.3.2' -// implementation 'ch.qos.logback:logback-classic:1.2.6' implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' - implementation('com.github.tronprotocol.java-tron:framework:GreatVoyage-v4.7.7') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - exclude group: "ch.qos.logback", module: "logback-classic" + if (rootProject.archInfo.isArm64) { + implementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.1-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + exclude group: "ch.qos.logback", module: "logback-classic" + } + } else { + implementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.1-x86_64') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + exclude group: "ch.qos.logback", module: "logback-classic" + } } implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') implementation(project(":toolkit")) { diff --git a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java index dbc0f95e..c7c91bb6 100644 --- a/tools/stress_test/src/main/java/org/tron/BroadcastTx.java +++ b/tools/stress_test/src/main/java/org/tron/BroadcastTx.java @@ -14,7 +14,6 @@ import org.tron.core.config.args.Args; import org.tron.core.net.TronNetDelegate; import org.tron.core.net.TronNetService; -import org.tron.core.services.RpcApiService; import org.tron.trident.core.ApiWrapper; import org.tron.trident.core.exceptions.IllegalException; import org.tron.trident.proto.Chain.Block; @@ -105,10 +104,18 @@ public Integer call() throws IOException, InterruptedException, IllegalException Args.getInstance().setNodeListenPort(nodeListenPort); logger.info("rpc.port: {}, node.listen.port {}", rpcPort, nodeListenPort); Args.getInstance().setOpenHistoryQueryWhenLiteFN(true); + Args.getInstance().setRpcEnable(true); + Args.getInstance().setRpcSolidityEnable(false); + Args.getInstance().setRpcPBFTEnable(false); + Args.getInstance().setFullNodeHttpEnable(false); + Args.getInstance().setSolidityNodeHttpEnable(false); + Args.getInstance().setPBFTHttpEnable(false); + Args.getInstance().setJsonRpcHttpFullNodeEnable(false); + Args.getInstance().setJsonRpcHttpSolidityNodeEnable(false); + Args.getInstance().setJsonRpcHttpPBFTNodeEnable(false); context = new TronApplicationContext(DefaultConfig.class); app = ApplicationFactory.create(context); - app.addService(context.getBean(RpcApiService.class)); app.startup(); String url = String.format("%s:%d", "127.0.0.1", diff --git a/tools/stress_test/src/main/java/org/tron/CollectAddress.java b/tools/stress_test/src/main/java/org/tron/CollectAddress.java index 8f57d2c9..1db8019c 100644 --- a/tools/stress_test/src/main/java/org/tron/CollectAddress.java +++ b/tools/stress_test/src/main/java/org/tron/CollectAddress.java @@ -1,7 +1,5 @@ package org.tron; -import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; - import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; import com.typesafe.config.Config; @@ -130,7 +128,7 @@ private Set getAddressListFromDB(String dbPath, int totalNumber) throws IOException, RocksDBException { Set addressList = new HashSet<>(); String srcDir = dbPath + File.separator + "database"; - DBInterface accountStore = DbTool.getDB(srcDir, ACCOUNT_STORE); + DBInterface accountStore = DbTool.getDB(srcDir, "account"); DBIterator iterator = accountStore.iterator(); for (iterator.seekToFirst(); iterator.valid(); iterator.next()) { addressList.add(ByteString.copyFrom(iterator.getKey())); diff --git a/tools/toolkit/README.md b/tools/toolkit/README.md index 082fb589..3dcd4c4d 100644 --- a/tools/toolkit/README.md +++ b/tools/toolkit/README.md @@ -23,11 +23,11 @@ expenses. - `convert`: Covert leveldb to rocksdb. - `lite`: Split lite data for java-tron. - `cp, copy`: Quick copy leveldb or rocksdb data. -- `root`: compute merkle root for tiny db. NOTE: large db may GC overhead -limit exceeded. +- `root`: compute merkle root for tiny db. NOTE: large db may GC overhead limit exceeded. - `fork`: Modify the database of java-tron for shadow fork testing. +- `query`: Query the latest vote and reward information from the database. -## DB Archive +## DB Archive(Requires x86 + LevelDB) DB archive provides the ability to reformat the manifest according to the current `database`, parameters are compatible with the previous `ArchiveManifest`. @@ -50,8 +50,7 @@ DB archive provides the ability to reformat the manifest according to the curren java -jar Toolkit.jar db archive -m 128 #4. specify optimization only when Manifest exceeds 128M ``` - -## DB Convert +## DB Convert(Requires x86 + LevelDB) DB convert provides a helper which can convert LevelDB data to RocksDB data, parameters are compatible with previous `DBConvert`. @@ -59,15 +58,13 @@ DB convert provides a helper which can convert LevelDB data to RocksDB data, par - ``: Input path for leveldb, default: output-directory/database. - ``: Output path for rocksdb, default: output-directory-dst/database. -- `--safe`: In safe mode, read data from leveldb then put into rocksdb, it's a very time-consuming procedure. If not, just change engine.properties from leveldb to rocksdb, rocksdb - is compatible with leveldb for the current version. This may not be the case in the future, default: false. - `-h | --help`: Provide the help info. ### Examples: ```shell script # full command - java -jar Toolkit.jar db convert [-h] [--safe] + java -jar Toolkit.jar db convert [-h] # examples java -jar Toolkit.jar db convert output-directory/database /tmp/database ``` @@ -91,7 +88,7 @@ DB copy provides a helper which can copy LevelDB or RocksDB data quickly on the java -jar Toolkit.jar db cp output-directory/database /tmp/databse ``` -## DB Lite +## DB Lite(LevelDB unavailable on ARM) DB lite provides lite database, parameters are compatible with previous `LiteFullNodeTool`. @@ -159,7 +156,7 @@ Execute move command. java -jar Toolkit.jar db mv -c main_net_config.conf -d /data/tron/output-directory ``` -## DB Root +## DB Root(LevelDB unavailable on ARM) DB root provides a helper which can compute merkle root for tiny db. @@ -171,7 +168,6 @@ NOTE: large db may GC overhead limit exceeded. - `--db`: db name. - `-h | --help`: provide the help info - ## DB Fork DB fork tool can help launch a private java-tron FullNode or network based on the state of public chain database to support shadow fork testing. diff --git a/tools/toolkit/build.gradle b/tools/toolkit/build.gradle index 450a14ca..4ba8b768 100644 --- a/tools/toolkit/build.gradle +++ b/tools/toolkit/build.gradle @@ -4,10 +4,13 @@ plugins { id 'application' id 'checkstyle' id 'com.github.johnrengelman.shadow' version '7.1.2' + id "org.sonarqube" version "2.6" } +def springVersion = "5.3.39" + sourceCompatibility = JavaVersion.VERSION_1_8 -targetCompatibility = JavaVersion.VERSION_1_8 +targetCompatibility = JavaVersion.current() [compileJava, compileTestJava]*.options*.encoding = 'UTF-8' repositories { @@ -17,6 +20,13 @@ repositories { maven { url 'https://jitpack.io' } } +// Force RocksDB version based on architecture (same as java-tron/platform module) +configurations.all { + resolutionStrategy { + force "org.rocksdb:rocksdbjni:${rootProject.archInfo.requires.RocksdbVersion}" + } +} + tasks.named('jar') { enabled = false dependsOn shadowJar @@ -29,35 +39,67 @@ shadowJar { mergeServiceFiles() } +sourceSets { + main { + java.srcDirs = rootProject.archInfo.sourceSets.main.java.srcDirs + } + test { + java.srcDirs = rootProject.archInfo.sourceSets.test.java.srcDirs + } +} + + dependencies { testImplementation 'junit:junit:4.13.2' - testImplementation('com.github.tronprotocol.java-tron:framework:GreatVoyage-v4.7.7') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - } - testImplementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') + implementation group: 'org.springframework', name: 'spring-context', version: "${springVersion}" implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' implementation group: 'com.typesafe', name: 'config', version: '1.3.2' implementation group: 'me.tongfei', name: 'progressbar', version: '0.9.3' - implementation group: 'org.bouncycastle', name: 'bcprov-jdk15on', version: '1.69' - implementation group: 'org.rocksdb', name: 'rocksdbjni', version: '5.15.10' - implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' - implementation 'io.github.tronprotocol:leveldb:1.18.2' - - implementation('com.github.tronprotocol.java-tron:chainbase:GreatVoyage-v4.7.7') { - exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' - exclude group: "pull-parser", module: "pull-parser" - } - implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') - implementation group: 'info.picocli', name: 'picocli', version: '4.6.3' + implementation group: 'org.bouncycastle', name: 'bcprov-jdk18on', version: '1.79' - compileOnly 'org.projectlombok:lombok:1.18.12' - annotationProcessor 'org.projectlombok:lombok:1.18.12' - testCompileOnly 'org.projectlombok:lombok:1.18.12' - testAnnotationProcessor 'org.projectlombok:lombok:1.18.12' + compileOnly 'org.projectlombok:lombok:1.18.34' + annotationProcessor 'org.projectlombok:lombok:1.18.34' + testCompileOnly 'org.projectlombok:lombok:1.18.34' + testAnnotationProcessor 'org.projectlombok:lombok:1.18.34' implementation 'com.alibaba:fastjson:1.2.83' + implementation group: 'org.slf4j', name: 'slf4j-api', version: '1.7.25' + implementation group: 'org.slf4j', name: 'jcl-over-slf4j', version: '1.7.25' + implementation "org.apache.commons:commons-lang3:3.4" + implementation "org.apache.commons:commons-collections4:4.1" + implementation platform('com.fasterxml.jackson:jackson-bom:2.14.0') + + if (rootProject.archInfo.isArm64) { + // ARM64 + testImplementation('com.github.warku123.java-tron:framework:GreatVoyage-v4.8.1-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.warku123.java-tron:chainbase:GreatVoyage-v4.8.1-arm') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation 'com.github.warku123.java-tron:platform:GreatVoyage-v4.8.1-arm' + } else { + // x86 + testImplementation('com.github.federico2014.java-tron:framework:GreatVoyage-v4.8.1-x86_64') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.federico2014.java-tron:chainbase:GreatVoyage-v4.8.1-x86_64') { + exclude group: 'com.fasterxml.jackson', module: 'jackson-bom' + exclude group: "pull-parser", module: "pull-parser" + } + implementation('com.github.federico2014.java-tron:platform:GreatVoyage-v4.8.1-x86_64') { + exclude(group: 'org.fusesource.leveldbjni', module: 'leveldbjni-all') + exclude(group: 'io.github.tronprotocol', module: 'zksnark-java-sdk') + exclude(group: 'commons-io', module: 'commons-io') + } + implementation 'io.github.tronprotocol:leveldbjni-all:1.18.2' + implementation 'io.github.tronprotocol:leveldb:1.18.2' + } + } application { @@ -95,3 +137,20 @@ task checkstyleToolkitTest(type: Checkstyle) { } check.dependsOn checkstyleToolkitMain, checkstyleToolkitTest + +test { + testLogging { + exceptionFormat = 'full' + } + + if (rootProject.archInfo.isArm64) { + exclude 'org/tron/plugins/leveldb/**' + filter { + excludeTestsMatching '*.*leveldb*' + excludeTestsMatching '*.*Leveldb*' + excludeTestsMatching '*.*LevelDB*' + excludeTestsMatching '*.*LevelDb*' + excludeTestsMatching '*.*Archive*' + } + } +} diff --git a/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java b/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java new file mode 100644 index 00000000..b7848cf4 --- /dev/null +++ b/tools/toolkit/src/main/java/arm/org/tron/plugins/ArchiveManifest.java @@ -0,0 +1,29 @@ +package org.tron.plugins; + +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +public class ArchiveManifest { + + public static void main(String[] args) { + int exitCode = run(args); + System.exit(exitCode); + } + + public static int run(String[] args) { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + System.out.println(tips); + logger.warn(tips); + return 0; + } + +} diff --git a/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java b/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java new file mode 100644 index 00000000..03c52f33 --- /dev/null +++ b/tools/toolkit/src/main/java/arm/org/tron/plugins/DbArchive.java @@ -0,0 +1,50 @@ +package org.tron.plugins; + +import java.util.concurrent.Callable; +import lombok.extern.slf4j.Slf4j; +import org.tron.common.arch.Arch; +import picocli.CommandLine; +import picocli.CommandLine.Option; + +/** + * ARM architecture only supports RocksDB, + * which does not require manifest rebuilding (manifest rebuilding is a LevelDB-only feature). + * This command is not supported but retained for compatibility. + **/ +@Slf4j(topic = "archive") +@CommandLine.Command(name = "archive", description = "A helper to rewrite leveldb manifest.") +public class DbArchive implements Callable { + + @CommandLine.Spec + CommandLine.Model.CommandSpec spec; + @Option(names = {"-d", "--database-directory"}, + defaultValue = "output-directory/database", + description = "java-tron database directory. Default: ${DEFAULT-VALUE}") + private String databaseDirectory; + + @Option(names = {"-b", "--batch-size"}, + defaultValue = "80000", + description = "deal manifest batch size. Default: ${DEFAULT-VALUE}") + private int maxBatchSize; + + @Option(names = {"-m", "--manifest-size"}, + defaultValue = "0", + description = "manifest min size(M) to archive. Default: ${DEFAULT-VALUE}") + private int maxManifestSize; + + @Option(names = {"-h", "--help"}) + private boolean help; + + + @Override + public Integer call() throws Exception { + String tips = String.format( + "%s architecture only supports RocksDB, which does not require manifest rebuilding " + + "(manifest rebuilding is a LevelDB-only feature).", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.warn(tips); + return 0; + } + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/Db.java b/tools/toolkit/src/main/java/common/org/tron/plugins/Db.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/Db.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/Db.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java similarity index 89% rename from tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java index a75b235b..bcf6e1e7 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbConvert.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbConvert.java @@ -20,6 +20,7 @@ import org.rocksdb.RocksDBException; import org.rocksdb.RocksIterator; import org.rocksdb.Status; +import org.tron.common.arch.Arch; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; @@ -49,20 +50,19 @@ public class DbConvert implements Callable { description = "Output path for rocksdb. Default: ${DEFAULT-VALUE}") private File dest; - @CommandLine.Option(names = {"--safe"}, - description = "In safe mode, read data from leveldb then put rocksdb." - + "If not, just change engine.properties from leveldb to rocksdb," - + "rocksdb is compatible with leveldb for current version." - + "This may not be the case in the future." - + "Default: ${DEFAULT-VALUE}") - private boolean safe; - @CommandLine.Option(names = {"-h", "--help"}) private boolean help; @Override public Integer call() throws Exception { + if (Arch.isArm64()) { + String tips = String.format("This command is not supported on %s architecture.", + Arch.getOsArch()); + spec.commandLine().getErr().println(spec.commandLine().getColorScheme().errorText(tips)); + logger.error(tips); + return 0; + } if (help) { spec.commandLine().usage(System.out); return 0; @@ -95,12 +95,12 @@ public Integer call() throws Exception { final long time = System.currentTimeMillis(); List services = new ArrayList<>(); files.forEach(f -> services.add( - new DbConverter(src.getPath(), dest.getPath(), f.getName(), safe))); + new DbConverter(src.getPath(), dest.getPath(), f.getName()))); cpList.forEach(f -> services.add( new DbConverter( Paths.get(src.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), Paths.get(dest.getPath(), DBUtils.CHECKPOINT_DB_V2).toString(), - f.getName(), safe))); + f.getName()))); List fails = ProgressBar.wrap(services.stream(), "convert task").parallel().map( dbConverter -> { try { @@ -140,15 +140,12 @@ static class DbConverter implements Converter { private long srcDbValueSum = 0L; private long dstDbValueSum = 0L; - private boolean safe; - - public DbConverter(String srcDir, String dstDir, String name, boolean safe) { + public DbConverter(String srcDir, String dstDir, String name) { this.srcDir = srcDir; this.dstDir = dstDir; this.dbName = name; this.srcDbPath = Paths.get(this.srcDir, name); this.dstDbPath = Paths.get(this.dstDir, name); - this.safe = safe; } @Override @@ -178,23 +175,15 @@ public boolean doConvert() throws Exception { FileUtils.createDirIfNotExists(dstDir); logger.info("Convert database {} start", this.dbName); - if (safe) { - convertLevelToRocks(); - compact(); - } else { - FileUtils.copyDir(Paths.get(srcDir), Paths.get(dstDir), dbName); - } + convertLevelToRocks(); + compact(); + boolean result = check() && createEngine(dstDbPath.toString()); long etime = System.currentTimeMillis(); if (result) { - if (safe) { - logger.info("Convert database {} successful end with {} key-value {} minutes", - this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); - } else { - logger.info("Convert database {} successful end {} minutes", - this.dbName, (etime - startTime) / 1000.0 / 60); - } + logger.info("Convert database {} successful end with {} key-value {} minutes", + this.dbName, this.srcDbKeyCount, (etime - startTime) / 1000.0 / 60); } else { logger.info("Convert database {} failure", this.dbName); @@ -234,8 +223,8 @@ private void batchInsert(RocksDB rocks, List keys, List values) * @throws Exception RocksDBException */ private void write(RocksDB rocks, org.rocksdb.WriteBatch batch) throws Exception { - try { - rocks.write(new org.rocksdb.WriteOptions(), batch); + try (org.rocksdb.WriteOptions writeOptions = new org.rocksdb.WriteOptions()) { + rocks.write(writeOptions, batch); } catch (RocksDBException e) { // retry if (maybeRetry(e)) { @@ -270,7 +259,8 @@ public void convertLevelToRocks() throws Exception { JniDBFactory.pushMemoryPool(1024 * 1024); try ( DB level = DBUtils.newLevelDb(srcDbPath); - RocksDB rocks = DBUtils.newRocksDbForBulkLoad(dstDbPath); + org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(true, dbName); + RocksDB rocks = RocksDB.open(options, this.dstDbPath.toString()); DBIterator levelIterator = level.iterator( new org.iq80.leveldb.ReadOptions().fillCache(false))) { @@ -302,7 +292,8 @@ private void compact() throws RocksDBException { if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.dbName)) { return; } - try (RocksDB rocks = DBUtils.newRocksDb(this.dstDbPath)) { + try (org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(false, dbName); + RocksDB rocks = RocksDB.open(options, this.dstDbPath.toString())) { logger.info("compact database {} start", this.dbName); rocks.compactRange(); logger.info("compact database {} end", this.dbName); @@ -310,11 +301,9 @@ private void compact() throws RocksDBException { } private boolean check() throws RocksDBException { - if (!safe) { - return true; - } try ( - RocksDB rocks = DBUtils.newRocksDbReadOnly(this.dstDbPath); + org.rocksdb.Options options = DBUtils.newDefaultRocksDbOptions(false, dbName); + RocksDB rocks = RocksDB.openReadOnly(options, this.dstDbPath.toString()); org.rocksdb.ReadOptions r = new org.rocksdb.ReadOptions().setFillCache(false); RocksIterator rocksIterator = rocks.newIterator(r)) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java index 5a4e5969..7558d764 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbCopy.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbCopy.java @@ -22,7 +22,7 @@ exitCodeListHeading = "Exit Codes:%n", exitCodeList = { "0:Successful", - "n:Internal error: exception occurred, please check toolkit.log"}) + "n:Internal error: exception occurred,please check toolkit.log"}) public class DbCopy implements Callable { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbFork.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/DbFork.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbFork.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbLite.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java similarity index 99% rename from tools/toolkit/src/main/java/org/tron/plugins/DbLite.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java index eb11883c..5ce2a445 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbLite.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbLite.java @@ -656,12 +656,13 @@ private boolean isLite(String databaseDir) throws RocksDBException, IOException private long getSecondBlock(String databaseDir) throws RocksDBException, IOException { long num = 0; DBInterface sourceBlockIndexDb = DbTool.getDB(databaseDir, BLOCK_INDEX_DB_NAME); - DBIterator iterator = sourceBlockIndexDb.iterator(); - iterator.seek(ByteArray.fromLong(1)); - if (iterator.hasNext()) { - num = Longs.fromByteArray(iterator.getKey()); + try (DBIterator iterator = sourceBlockIndexDb.iterator()) { + iterator.seek(ByteArray.fromLong(1)); + if (iterator.hasNext()) { + num = Longs.fromByteArray(iterator.getKey()); + } + return num; } - return num; } private DBInterface getCheckpointDb(String sourceDir) throws IOException, RocksDBException { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbMove.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbMove.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/DbMove.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbMove.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/DbQuery.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbQuery.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java b/tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java similarity index 84% rename from tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java index 7c33219e..45854bbe 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbRoot.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/DbRoot.java @@ -67,16 +67,24 @@ public Integer call() throws Exception { .errorText("Specify at least one exit database: --db dbName.")); return 404; } - List task = ProgressBar.wrap(dbs.stream(), "root task").parallel() - .map(this::calcMerkleRoot).collect(Collectors.toList()); - task.forEach(this::printInfo); - int code = (int) task.stream().filter(r -> r.code == 1).count(); - if (code > 0) { + try { + List task = ProgressBar.wrap(dbs.stream(), "root task").parallel() + .map(this::calcMerkleRoot).collect(Collectors.toList()); + task.forEach(this::printInfo); + int code = (int) task.stream().filter(r -> r.code == 1).count(); + if (code > 0) { + spec.commandLine().getErr().println(spec.commandLine().getColorScheme() + .errorText("There are some errors, please check toolkit.log for detail.")); + } + spec.commandLine().getOut().println("root task done."); + return code; + } catch (Exception e) { + logger.error("{}", e); spec.commandLine().getErr().println(spec.commandLine().getColorScheme() - .errorText("There are some errors, please check toolkit.log for detail.")); + .errorText(e.getMessage())); + spec.commandLine().usage(System.out); + return 1; } - spec.commandLine().getOut().println("root task done."); - return code; } private Ret calcMerkleRoot(String name) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/Toolkit.java b/tools/toolkit/src/main/java/common/org/tron/plugins/Toolkit.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/Toolkit.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/Toolkit.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java similarity index 97% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java index 3422c36c..2922d110 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/ByteArray.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/ByteArray.java @@ -59,7 +59,7 @@ public static int compareUnsigned(byte[] a, byte[] b) { if (b == null) { return 1; } - int minLen = Math.min(a.length, b.length); + int minLen = StrictMath.min(a.length, b.length); for (int i = 0; i < minLen; ++i) { int aVal = a[i] & 0xFF; int bVal = b[i] & 0xFF; diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/Constant.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/Constant.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/CryptoUitls.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/CryptoUitls.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/CryptoUitls.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java similarity index 70% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java index 19547e2b..6eb097cb 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/DBUtils.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/DBUtils.java @@ -14,10 +14,9 @@ import org.rocksdb.BloomFilter; import org.rocksdb.ComparatorOptions; import org.rocksdb.Options; -import org.rocksdb.RocksDB; -import org.rocksdb.RocksDBException; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForLevelDB; -import org.tron.plugins.comparator.MarketOrderPriceComparatorForRockDB; +import org.tron.common.arch.Arch; +import org.tron.common.utils.MarketOrderPriceComparatorForLevelDB; +import org.tron.common.utils.MarketOrderPriceComparatorForRocksDB; import org.tron.protos.Protocol; public class DBUtils { @@ -65,6 +64,7 @@ static Operator valueOf(byte b) { public static final String ROCKSDB = "ROCKSDB"; public static DB newLevelDb(Path db) throws IOException { + Arch.throwIfUnsupportedArm64Exception(LEVELDB); File file = db.toFile(); org.iq80.leveldb.Options dbOptions = newDefaultLevelDbOptions(); if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(file.getName())) { @@ -86,7 +86,23 @@ public static org.iq80.leveldb.Options newDefaultLevelDbOptions() { return dbOptions; } - private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { + /** + * Creates a new RocksDB Options. + * + *

CRITICAL: Must be closed after use to prevent native memory leaks. + * Use try-with-resources. + * + * 

{@code
+   * try (Options options = newDefaultRocksDbOptions(false, name)) {
+   *     // do something
+   * }
+   * }
+ * + * @param forBulkLoad if true, optimizes for bulk loading + * @param name db name + * @return a new Options instance that must be closed + */ + public static Options newDefaultRocksDbOptions(boolean forBulkLoad, String name) { Options options = new Options(); options.setCreateIfMissing(true); options.setIncreaseParallelism(1); @@ -95,7 +111,8 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { options.setTargetFileSizeBase(64 * 1024 * 1024); options.setTargetFileSizeMultiplier(1); options.setMaxBytesForLevelBase(512 * 1024 * 1024); - options.setMaxBackgroundCompactions(Math.max(1, Runtime.getRuntime().availableProcessors())); + options.setMaxBackgroundCompactions(StrictMath.max( + 1, Runtime.getRuntime().availableProcessors())); options.setLevel0FileNumCompactionTrigger(4); options.setLevelCompactionDynamicLevelBytes(true); final BlockBasedTableConfig tableCfg; @@ -108,35 +125,10 @@ private static Options newDefaultRocksDbOptions(boolean forBulkLoad) { if (forBulkLoad) { options.prepareForBulkLoad(); } - return options; - } - - public static RocksDB newRocksDb(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(false)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.open(options, db.toString()); - } - } - - public static RocksDB newRocksDbForBulkLoad(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(true)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.open(options, db.toString()); - } - } - - - public static RocksDB newRocksDbReadOnly(Path db) throws RocksDBException { - try (Options options = newDefaultRocksDbOptions(false)) { - if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(db.getFileName().toString())) { - options.setComparator(new MarketOrderPriceComparatorForRockDB(new ComparatorOptions())); - } - return RocksDB.openReadOnly(options, db.toString()); + if (MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(name)) { + options.setComparator(new MarketOrderPriceComparatorForRocksDB(new ComparatorOptions())); } + return options; } public static String simpleDecode(byte[] bytes) { diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/FileUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/FileUtils.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/FileUtils.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/FileUtils.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/HttpSelfFormatFieldName.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/HttpSelfFormatFieldName.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/JsonFormat.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/JsonFormat.java diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java new file mode 100644 index 00000000..9bcfd5e7 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MarketUtils.java @@ -0,0 +1,68 @@ +package org.tron.plugins.utils; + +public class MarketUtils { + + public static final int TOKEN_ID_LENGTH = ByteArray + .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 + + + + /** + * In order to avoid the difference between the data of same key stored and fetched by hashMap and + * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of + * sellTokenQuantity and buyTokenQuantity. + */ + public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + long sellTokenQuantity, long buyTokenQuantity) { + + byte[] sellTokenQuantityBytes; + byte[] buyTokenQuantityBytes; + + // cal the GCD + long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); + if (gcd == 0) { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); + } else { + sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); + buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); + } + + return doCreatePairPriceKey(sellTokenId, buyTokenId, + sellTokenQuantityBytes, buyTokenQuantityBytes); + } + + public static long findGCD(long number1, long number2) { + if (number1 == 0 || number2 == 0) { + return 0; + } + return calGCD(number1, number2); + } + + private static long calGCD(long number1, long number2) { + if (number2 == 0) { + return number1; + } + return calGCD(number2, number1 % number2); + } + + + private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, + byte[] sellTokenQuantity, byte[] buyTokenQuantity) { + byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + + sellTokenQuantity.length + buyTokenQuantity.length]; + + System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); + System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); + System.arraycopy(sellTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, + sellTokenQuantity.length); + System.arraycopy(buyTokenQuantity, 0, result, + TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, + buyTokenQuantity.length); + + return result; + } + + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/MerkleRoot.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/MerkleRoot.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/MerkleRoot.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/Sha256Hash.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/Sha256Hash.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/Sha256Hash.java diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java new file mode 100644 index 00000000..13a195f9 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBInterface.java @@ -0,0 +1,39 @@ +package org.tron.plugins.utils.db; + +import java.io.Closeable; +import java.io.IOException; + + +public interface DBInterface extends Closeable { + + byte[] get(byte[] key); + + void put(byte[] key, byte[] value); + + void delete(byte[] key); + + /** + * Returns an iterator over the database. + * + *

CRITICAL: The returned iterator holds native resources and MUST be closed + * after use to prevent memory leaks. It is strongly recommended to use a try-with-resources + * statement. + * + *

Example of correct usage: + * 

{@code
+   * try (DBIterator iterator = db.iterator()) {
+   *  // do something
+   * }
+   * }
+ * + * @return a new database iterator that must be closed. + */ + DBIterator iterator(); + + long size() throws IOException; + + void close() throws IOException; + + String getName(); + +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DBIterator.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java similarity index 83% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java index 429025e8..d9962394 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DbTool.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/DbTool.java @@ -9,6 +9,7 @@ import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.rocksdb.RocksDBException; +import org.tron.common.arch.Arch; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; @@ -18,8 +19,8 @@ public class DbTool { private static final String KEY_ENGINE = "ENGINE"; private static final String ENGINE_FILE = "engine.properties"; - private static final String FILE_SEPARATOR = File.separator; private static final String ROCKSDB = "ROCKSDB"; + private static final String LEVELDB = "LEVELDB"; private static final Map dbMap = Maps.newConcurrentMap(); @@ -162,10 +163,10 @@ public static void close() { } private static DbType getDbType(String sourceDir, String dbName) { - String engineFile = String.format("%s%s%s%s%s", sourceDir, FILE_SEPARATOR, - dbName, FILE_SEPARATOR, ENGINE_FILE); + String engineFile = Paths.get(sourceDir, dbName, ENGINE_FILE).toString(); if (!new File(engineFile).exists()) { - return DbType.LevelDB; + // For arm64, only support RocksDB + return Arch.isArm64() ? DbType.RocksDB : DbType.LevelDB; } String engine = FileUtils.readProperty(engineFile, KEY_ENGINE); if (engine.equalsIgnoreCase(ROCKSDB)) { @@ -175,13 +176,22 @@ private static DbType getDbType(String sourceDir, String dbName) { } } - private static LevelDBImpl openLevelDb(Path db, String name) throws IOException { - return new LevelDBImpl(DBUtils.newLevelDb(db), name); + public static LevelDBImpl openLevelDb(Path db, String name) throws IOException { + LevelDBImpl leveldb = new LevelDBImpl(DBUtils.newLevelDb(db), name); + tryInitEngineFile(db, LEVELDB); + return leveldb; } - private static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { - return new RocksDBImpl(DBUtils.newRocksDb(db), name); + public static RocksDBImpl openRocksDb(Path db, String name) throws RocksDBException { + RocksDBImpl rocksdb = new RocksDBImpl(db, name); + tryInitEngineFile(db, ROCKSDB); + return rocksdb; } - + private static void tryInitEngineFile(Path db, String engine) { + String engineFile = Paths.get(db.toString(), ENGINE_FILE).toString(); + if (FileUtils.createFileIfNotExists(engineFile)) { + FileUtils.writeProperty(engineFile, KEY_ENGINE, engine); + } + } } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java similarity index 83% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java index 511f4dfd..1c7f22ef 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBImpl.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBImpl.java @@ -40,8 +40,11 @@ public DBIterator iterator() { } @Override - public long size() { - return Streams.stream(leveldb.iterator()).count(); + public long size() throws IOException { + try (DBIterator iterator = this.iterator()) { + iterator.seekToFirst(); + return Streams.stream(iterator).count(); + } } @Override diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java similarity index 100% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/LevelDBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/LevelDBIterator.java diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java similarity index 76% rename from tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java rename to tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java index d3e17d91..17ecca4a 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RockDBIterator.java +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RockDBIterator.java @@ -2,14 +2,19 @@ import java.io.IOException; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import org.rocksdb.ReadOptions; import org.rocksdb.RocksIterator; public class RockDBIterator implements DBIterator { private final RocksIterator iterator; + private final ReadOptions readOptions; + private final AtomicBoolean closed = new AtomicBoolean(false); - public RockDBIterator(RocksIterator iterator) { + public RockDBIterator(RocksIterator iterator, ReadOptions readOptions) { this.iterator = iterator; + this.readOptions = readOptions; } @Override @@ -72,6 +77,9 @@ public byte[] setValue(byte[] value) { @Override public void close() throws IOException { - iterator.close(); + if (closed.compareAndSet(false, true)) { + readOptions.close(); + iterator.close(); + } } } diff --git a/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java new file mode 100644 index 00000000..236d0a84 --- /dev/null +++ b/tools/toolkit/src/main/java/common/org/tron/plugins/utils/db/RocksDBImpl.java @@ -0,0 +1,97 @@ +package org.tron.plugins.utils.db; + +import com.google.common.collect.Streams; +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; +import org.rocksdb.Options; +import org.rocksdb.ReadOptions; +import org.rocksdb.RocksDB; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; + +public class RocksDBImpl implements DBInterface { + + private final RocksDB rocksDB; + + @Getter + private final String name; + private final AtomicBoolean closed = new AtomicBoolean(false); + private Options options = null; + + public RocksDBImpl(Path path, String name) throws RocksDBException { + try { + this.options = DBUtils.newDefaultRocksDbOptions(false, name); + this.name = name; + this.rocksDB = RocksDB.open(options, path.toString()); + } catch (RocksDBException e) { + if (this.options != null) { + this.options.close(); + } + throw e; + } + } + + @Override + public byte[] get(byte[] key) { + throwIfClosed(); + try { + return rocksDB.get(key); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public void put(byte[] key, byte[] value) { + throwIfClosed(); + try { + rocksDB.put(key, value); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public void delete(byte[] key) { + throwIfClosed(); + try { + rocksDB.delete(key); + } catch (RocksDBException e) { + throw new RuntimeException(name, e); + } + } + + @Override + public DBIterator iterator() { + throwIfClosed(); + ReadOptions readOptions = new ReadOptions().setFillCache(false); + return new RockDBIterator(rocksDB.newIterator(readOptions), readOptions); + } + + @Override + public long size() throws IOException { + throwIfClosed(); + try (DBIterator iterator = this.iterator()) { + iterator.seekToFirst(); + return Streams.stream(iterator).count(); + } + } + + @Override + public void close() throws IOException { + if (closed.compareAndSet(false, true)) { + if (this.options != null) { + this.options.close(); + } + rocksDB.close(); + } + } + + private void throwIfClosed() { + if (closed.get()) { + throw new IllegalStateException("db " + name + " has been closed"); + } + } +} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java b/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java deleted file mode 100644 index 9cb12b49..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForLevelDB.java +++ /dev/null @@ -1,28 +0,0 @@ -package org.tron.plugins.comparator; - -import org.iq80.leveldb.DBComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForLevelDB implements DBComparator { - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public byte[] findShortestSeparator(byte[] start, byte[] limit) { - return new byte[0]; - } - - @Override - public byte[] findShortSuccessor(byte[] key) { - return new byte[0]; - } - - @Override - public int compare(byte[] o1, byte[] o2) { - return MarketUtils.comparePriceKey(o1, o2); - } - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java b/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java deleted file mode 100644 index 40d65516..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/comparator/MarketOrderPriceComparatorForRockDB.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.tron.plugins.comparator; - -import org.rocksdb.ComparatorOptions; -import org.rocksdb.DirectSlice; -import org.rocksdb.util.DirectBytewiseComparator; -import org.tron.plugins.utils.MarketUtils; - -public class MarketOrderPriceComparatorForRockDB extends DirectBytewiseComparator { - - public MarketOrderPriceComparatorForRockDB(final ComparatorOptions copt) { - super(copt); - } - - @Override - public String name() { - return "MarketOrderPriceComparator"; - } - - @Override - public int compare(final DirectSlice a, final DirectSlice b) { - return MarketUtils.comparePriceKey(convertDataToBytes(a), convertDataToBytes(b)); - } - - /** - * DirectSlice.data().array will throw UnsupportedOperationException. - * */ - public byte[] convertDataToBytes(DirectSlice directSlice) { - int capacity = directSlice.data().capacity(); - byte[] bytes = new byte[capacity]; - - for (int i = 0; i < capacity; i++) { - bytes[i] = directSlice.get(i); - } - - return bytes; - } - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java deleted file mode 100644 index a36fc8ad..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/MarketUtils.java +++ /dev/null @@ -1,148 +0,0 @@ -package org.tron.plugins.utils; - -import java.math.BigInteger; -import org.tron.plugins.utils.ByteArray; - -public class MarketUtils { - - public static final int TOKEN_ID_LENGTH = ByteArray - .fromString(Long.toString(Long.MAX_VALUE)).length; // 19 - - - - /** - * In order to avoid the difference between the data of same key stored and fetched by hashMap and - * levelDB, when creating the price key, we will find the GCD (Greatest Common Divisor) of - * sellTokenQuantity and buyTokenQuantity. - */ - public static byte[] createPairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - long sellTokenQuantity, long buyTokenQuantity) { - - byte[] sellTokenQuantityBytes; - byte[] buyTokenQuantityBytes; - - // cal the GCD - long gcd = findGCD(sellTokenQuantity, buyTokenQuantity); - if (gcd == 0) { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity); - } else { - sellTokenQuantityBytes = ByteArray.fromLong(sellTokenQuantity / gcd); - buyTokenQuantityBytes = ByteArray.fromLong(buyTokenQuantity / gcd); - } - - return doCreatePairPriceKey(sellTokenId, buyTokenId, - sellTokenQuantityBytes, buyTokenQuantityBytes); - } - - public static long findGCD(long number1, long number2) { - if (number1 == 0 || number2 == 0) { - return 0; - } - return calGCD(number1, number2); - } - - private static long calGCD(long number1, long number2) { - if (number2 == 0) { - return number1; - } - return calGCD(number2, number1 % number2); - } - - - private static byte[] doCreatePairPriceKey(byte[] sellTokenId, byte[] buyTokenId, - byte[] sellTokenQuantity, byte[] buyTokenQuantity) { - byte[] result = new byte[TOKEN_ID_LENGTH + TOKEN_ID_LENGTH - + sellTokenQuantity.length + buyTokenQuantity.length]; - - System.arraycopy(sellTokenId, 0, result, 0, sellTokenId.length); - System.arraycopy(buyTokenId, 0, result, TOKEN_ID_LENGTH, buyTokenId.length); - System.arraycopy(sellTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - sellTokenQuantity.length); - System.arraycopy(buyTokenQuantity, 0, result, - TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + buyTokenQuantity.length, - buyTokenQuantity.length); - - return result; - } - - - public static int comparePriceKey(byte[] o1, byte[] o2) { - //compare pair - byte[] pair1 = new byte[TOKEN_ID_LENGTH * 2]; - byte[] pair2 = new byte[TOKEN_ID_LENGTH * 2]; - - System.arraycopy(o1, 0, pair1, 0, TOKEN_ID_LENGTH * 2); - System.arraycopy(o2, 0, pair2, 0, TOKEN_ID_LENGTH * 2); - - int pairResult = ByteArray.compareUnsigned(pair1, pair2); - if (pairResult != 0) { - return pairResult; - } - - //compare price - byte[] getSellTokenQuantity1 = new byte[8]; - byte[] getBuyTokenQuantity1 = new byte[8]; - - byte[] getSellTokenQuantity2 = new byte[8]; - byte[] getBuyTokenQuantity2 = new byte[8]; - - int longByteNum = 8; - - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity1, 0, longByteNum); - System.arraycopy(o1, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity1, 0, longByteNum); - - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH, - getSellTokenQuantity2, 0, longByteNum); - System.arraycopy(o2, TOKEN_ID_LENGTH + TOKEN_ID_LENGTH + longByteNum, - getBuyTokenQuantity2, 0, longByteNum); - - long sellTokenQuantity1 = ByteArray.toLong(getSellTokenQuantity1); - long buyTokenQuantity1 = ByteArray.toLong(getBuyTokenQuantity1); - long sellTokenQuantity2 = ByteArray.toLong(getSellTokenQuantity2); - long buyTokenQuantity2 = ByteArray.toLong(getBuyTokenQuantity2); - - if ((sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) - && (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0)) { - return 0; - } - - if (sellTokenQuantity1 == 0 || buyTokenQuantity1 == 0) { - return -1; - } - - if (sellTokenQuantity2 == 0 || buyTokenQuantity2 == 0) { - return 1; - } - - return comparePrice(sellTokenQuantity1, buyTokenQuantity1, - sellTokenQuantity2, buyTokenQuantity2); - - } - - /** - * Note: the params should be the same token pair, or you should change the order. - * All the quantity should be bigger than 0. - * */ - public static int comparePrice(long price1SellQuantity, long price1BuyQuantity, - long price2SellQuantity, long price2BuyQuantity) { - try { - return Long.compare(Math.multiplyExact(price1BuyQuantity, price2SellQuantity), - Math.multiplyExact(price2BuyQuantity, price1SellQuantity)); - - } catch (ArithmeticException ex) { - // do nothing here, because we will use BigInteger to compute again - } - - BigInteger price1BuyQuantityBI = BigInteger.valueOf(price1BuyQuantity); - BigInteger price1SellQuantityBI = BigInteger.valueOf(price1SellQuantity); - BigInteger price2BuyQuantityBI = BigInteger.valueOf(price2BuyQuantity); - BigInteger price2SellQuantityBI = BigInteger.valueOf(price2SellQuantity); - - return price1BuyQuantityBI.multiply(price2SellQuantityBI) - .compareTo(price2BuyQuantityBI.multiply(price1SellQuantityBI)); - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java deleted file mode 100644 index 513e021c..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/DBInterface.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.tron.plugins.utils.db; - -import java.io.Closeable; -import java.io.IOException; - - -public interface DBInterface extends Closeable { - - byte[] get(byte[] key); - - void put(byte[] key, byte[] value); - - void delete(byte[] key); - - DBIterator iterator(); - - long size(); - - void close() throws IOException; - - String getName(); - -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java b/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java deleted file mode 100644 index 50957bbe..00000000 --- a/tools/toolkit/src/main/java/org/tron/plugins/utils/db/RocksDBImpl.java +++ /dev/null @@ -1,69 +0,0 @@ -package org.tron.plugins.utils.db; - -import java.io.IOException; -import lombok.Getter; -import org.rocksdb.RocksDBException; -import org.rocksdb.RocksIterator; - -public class RocksDBImpl implements DBInterface { - - private org.rocksdb.RocksDB rocksDB; - - @Getter - private final String name; - - public RocksDBImpl(org.rocksdb.RocksDB rocksDB, String name) { - this.rocksDB = rocksDB; - this.name = name; - } - - @Override - public byte[] get(byte[] key) { - try { - return rocksDB.get(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - return null; - } - - @Override - public void put(byte[] key, byte[] value) { - try { - rocksDB.put(key, value); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public void delete(byte[] key) { - try { - rocksDB.delete(key); - } catch (RocksDBException e) { - e.printStackTrace(); - } - } - - @Override - public DBIterator iterator() { - return new RockDBIterator(rocksDB.newIterator( - new org.rocksdb.ReadOptions().setFillCache(false))); - } - - @Override - public long size() { - RocksIterator iterator = rocksDB.newIterator(); - long size = 0; - for (iterator.seekToFirst(); iterator.isValid(); iterator.next()) { - size++; - } - iterator.close(); - return size; - } - - @Override - public void close() throws IOException { - rocksDB.close(); - } -} diff --git a/tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java rename to tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java index 7d638bcd..115144be 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/ArchiveManifest.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/ArchiveManifest.java @@ -35,6 +35,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -183,7 +184,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } diff --git a/tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java similarity index 98% rename from tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java rename to tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java index 2f117cd4..1088468f 100644 --- a/tools/toolkit/src/main/java/org/tron/plugins/DbArchive.java +++ b/tools/toolkit/src/main/java/x86/org/tron/plugins/DbArchive.java @@ -19,6 +19,7 @@ import org.iq80.leveldb.DB; import org.iq80.leveldb.Options; import org.iq80.leveldb.impl.Filename; +import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.FileUtils; import picocli.CommandLine; import picocli.CommandLine.Option; @@ -163,7 +164,7 @@ public boolean checkManifest(String dir) throws IOException { return false; } logger.info("CurrentName {}/{},size {} kb.", dir, currentName, current.length() / 1024); - if ("market_pair_price_to_order".equalsIgnoreCase(this.name)) { + if (DBUtils.MARKET_PAIR_PRICE_TO_ORDER.equalsIgnoreCase(this.name)) { logger.info("Db {} ignored.", this.name); return false; } @@ -194,5 +195,4 @@ public void doArchive() { } } } - } diff --git a/tools/toolkit/src/main/resources/logback.xml b/tools/toolkit/src/main/resources/logback.xml index dd2f2538..555edd23 100644 --- a/tools/toolkit/src/main/resources/logback.xml +++ b/tools/toolkit/src/main/resources/logback.xml @@ -53,5 +53,5 @@ - + diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java index 9e488a59..571fd8f5 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbCopyTest.java @@ -4,14 +4,25 @@ import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbCopyTest extends DbTest { @Test - public void testRun() { + public void testRunForLevelDB() throws RocksDBException, IOException { + init(DbTool.DbType.LevelDB); String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, - genarateTmpDir()}; + generateTmpDir()}; + Assert.assertEquals(0, cli.execute(args)); + } + + @Test + public void testRunForRocksDB() throws RocksDBException, IOException { + init(DbTool.DbType.RocksDB); + String[] args = new String[] { "db", "cp", INPUT_DIRECTORY, + generateTmpDir()}; Assert.assertEquals(0, cli.execute(args)); } @@ -32,7 +43,7 @@ public void testNotExist() { @Test public void testEmpty() throws IOException { String[] args = new String[] {"db", "cp", temporaryFolder.newFolder().toString(), - genarateTmpDir()}; + generateTmpDir()}; Assert.assertEquals(0, cli.execute(args)); } @@ -46,7 +57,7 @@ public void testDestIsExist() throws IOException { @Test public void testSrcIsFile() throws IOException { String[] args = new String[] {"db", "cp", temporaryFolder.newFile().toString(), - genarateTmpDir()}; + generateTmpDir()}; Assert.assertEquals(403, cli.execute(args)); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java index 1dea60b6..3eeb05fa 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbForkTest.java @@ -1,7 +1,33 @@ package org.tron.plugins; import static org.tron.plugins.DbFork.getActiveWitness; -import static org.tron.plugins.utils.Constant.*; +import static org.tron.plugins.utils.Constant.ACCOUNTS_KEY; +import static org.tron.plugins.utils.Constant.ACCOUNT_ADDRESS; +import static org.tron.plugins.utils.Constant.ACCOUNT_ASSET; +import static org.tron.plugins.utils.Constant.ACCOUNT_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_NAME; +import static org.tron.plugins.utils.Constant.ACCOUNT_OWNER; +import static org.tron.plugins.utils.Constant.ACCOUNT_STORE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_BALANCE; +import static org.tron.plugins.utils.Constant.ACCOUNT_TRC10_ID; +import static org.tron.plugins.utils.Constant.ACCOUNT_TYPE; +import static org.tron.plugins.utils.Constant.ACTIVE_WITNESSES; +import static org.tron.plugins.utils.Constant.ASSET_ISSUE_V2; +import static org.tron.plugins.utils.Constant.CONTRACT_STORE; +import static org.tron.plugins.utils.Constant.DYNAMIC_PROPERTY_STORE; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_HEADER_TIMESTAMP; +import static org.tron.plugins.utils.Constant.LATEST_BLOCK_TIMESTAMP; +import static org.tron.plugins.utils.Constant.MAINTENANCE_INTERVAL; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.MAINTENANCE_TIME_INTERVAL; +import static org.tron.plugins.utils.Constant.NEXT_MAINTENANCE_TIME; +import static org.tron.plugins.utils.Constant.STORAGE_ROW_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_ADDRESS; +import static org.tron.plugins.utils.Constant.WITNESS_KEY; +import static org.tron.plugins.utils.Constant.WITNESS_SCHEDULE_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_STORE; +import static org.tron.plugins.utils.Constant.WITNESS_URL; +import static org.tron.plugins.utils.Constant.WITNESS_VOTE; import com.google.common.primitives.Bytes; import com.google.protobuf.ByteString; @@ -36,13 +62,10 @@ public class DbForkTest { private DBInterface dynamicPropertiesStore; private DBInterface accountAssetStore; private DBInterface assetIssueV2Store; - private DBInterface contractStore; - private DBInterface storageRowStore; @Rule public final TemporaryFolder folder = new TemporaryFolder(); private String dbPath; - private String forkPath; public void createDir() { String srcDir = dbPath + File.separator + "database"; @@ -67,8 +90,6 @@ public void init() throws IOException, RocksDBException { dynamicPropertiesStore = DbTool.getDB(srcDir, Constant.DYNAMIC_PROPERTY_STORE); accountAssetStore = DbTool.getDB(srcDir, Constant.ACCOUNT_ASSET); assetIssueV2Store = DbTool.getDB(srcDir, Constant.ASSET_ISSUE_V2); - contractStore = DbTool.getDB(srcDir, Constant.CONTRACT_STORE); - storageRowStore = DbTool.getDB(srcDir, Constant.STORAGE_ROW_STORE); } public void close() { @@ -78,7 +99,7 @@ public void close() { @Test public void testDbFork() throws IOException, RocksDBException { dbPath = folder.newFolder().toString(); - forkPath = getConfig("fork.conf"); + String forkPath = getConfig("fork.conf"); createDir(); String[] args = new String[]{"-d", diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java index 3d3b9d5f..1d5771a0 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbLiteTest.java @@ -1,12 +1,12 @@ package org.tron.plugins; +import static org.tron.plugins.utils.PublicMethod.getRandomPrivateKey; + import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; import java.io.File; import java.io.IOException; -import java.net.URL; import java.nio.file.Paths; -import java.util.concurrent.TimeUnit; import lombok.extern.slf4j.Slf4j; import org.junit.After; import org.junit.Rule; @@ -18,11 +18,10 @@ import org.tron.common.config.DbBackupConfig; import org.tron.common.crypto.ECKey; import org.tron.common.utils.FileUtil; +import org.tron.common.utils.TimeoutInterceptor; import org.tron.common.utils.Utils; import org.tron.core.config.DefaultConfig; import org.tron.core.config.args.Args; -import org.tron.core.services.RpcApiService; -import org.tron.core.services.interfaceOnSolidity.RpcApiServiceOnSolidity; import org.tron.plugins.utils.PublicMethod; import picocli.CommandLine; @@ -47,14 +46,13 @@ public class DbLiteTest { public void startApp() { context = new TronApplicationContext(DefaultConfig.class); appTest = ApplicationFactory.create(context); - appTest.addService(context.getBean(RpcApiService.class)); - appTest.addService(context.getBean(RpcApiServiceOnSolidity.class)); appTest.startup(); String fullNode = String.format("%s:%d", "127.0.0.1", Args.getInstance().getRpcPort()); channelFull = ManagedChannelBuilder.forTarget(fullNode) .usePlaintext() + .intercept(new TimeoutInterceptor(5000)) .build(); blockingStubFull = WalletGrpc.newBlockingStub(channelFull); } @@ -64,18 +62,20 @@ public void startApp() { */ public void shutdown() throws InterruptedException { if (channelFull != null) { - channelFull.shutdown().awaitTermination(5, TimeUnit.SECONDS); + channelFull.shutdownNow(); } context.close(); } - public void init() throws IOException { + public void init(String dbType) throws IOException { dbPath = folder.newFolder().toString(); - Args.setParam(new String[]{"-d", dbPath, "-w", "--p2p-disable", "true"}, - getConfig("config-localtest.conf")); + Args.setParam(new String[] { + "-d", dbPath, "-w", "--p2p-disable", "true", "--storage-db-engine", dbType}, + "config-localtest.conf"); // allow account root Args.getInstance().setAllowAccountStateRoot(1); Args.getInstance().setRpcPort(PublicMethod.chooseRandomPort()); + Args.getInstance().setRpcEnable(true); databaseDir = Args.getInstance().getStorage().getDbDirectory(); // init dbBackupConfig to avoid NPE Args.getInstance().dbBackupConfig = DbBackupConfig.getInstance(); @@ -86,23 +86,22 @@ public void clear() { Args.clearParam(); } - void testTools(String dbType, int checkpointVersion) + public void testTools(String dbType, int checkpointVersion) throws InterruptedException, IOException { logger.info("dbType {}, checkpointVersion {}", dbType, checkpointVersion); dbPath = String.format("%s_%s_%d", dbPath, dbType, System.currentTimeMillis()); - init(); + init(dbType); final String[] argsForSnapshot = - new String[]{"-o", "split", "-t", "snapshot", "--fn-data-path", + new String[] {"-o", "split", "-t", "snapshot", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; final String[] argsForHistory = - new String[]{"-o", "split", "-t", "history", "--fn-data-path", + new String[] {"-o", "split", "-t", "history", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath}; final String[] argsForMerge = - new String[]{"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, + new String[] {"-o", "merge", "--fn-data-path", dbPath + File.separator + databaseDir, "--dataset-path", dbPath + File.separator + "history"}; - Args.getInstance().getStorage().setDbEngine(dbType); Args.getInstance().getStorage().setCheckpointVersion(checkpointVersion); DbLite.setRecentBlks(3); // start fullNode @@ -127,15 +126,15 @@ void testTools(String dbType, int checkpointVersion) File database = new File(Paths.get(dbPath, databaseDir).toString()); if (!database.renameTo(new File(Paths.get(dbPath, databaseDir + "_bak").toString()))) { throw new RuntimeException( - String.format("rename %s to %s failed", database.getPath(), - Paths.get(dbPath, databaseDir))); + String.format("rename %s to %s failed", database.getPath(), + Paths.get(dbPath, databaseDir))); } // change snapshot to the new database File snapshot = new File(Paths.get(dbPath, "snapshot").toString()); if (!snapshot.renameTo(new File(Paths.get(dbPath, databaseDir).toString()))) { throw new RuntimeException( - String.format("rename snapshot to %s failed", - Paths.get(dbPath, databaseDir))); + String.format("rename snapshot to %s failed", + Paths.get(dbPath, databaseDir))); } // start and validate the snapshot startApp(); @@ -159,10 +158,10 @@ private void generateSomeTransactions(int during) { ECKey ecKey2 = new ECKey(Utils.getRandom()); byte[] address = ecKey2.getAddress(); - String sunPri = "cba92a516ea09f620a16ff7ee95ce0df1d56550a8babe9964981a7144c8a784a"; + String sunPri = getRandomPrivateKey(); byte[] sunAddress = PublicMethod.getFinalAddress(sunPri); PublicMethod.sendcoin(address, 1L, - sunAddress, sunPri, blockingStubFull); + sunAddress, sunPri, blockingStubFull); try { Thread.sleep(sleepOnce); } catch (InterruptedException e) { @@ -173,9 +172,4 @@ private void generateSomeTransactions(int during) { } } } - - private static String getConfig(String config) { - URL path = DbLiteTest.class.getClassLoader().getResource(config); - return path == null ? null : path.getPath(); - } } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java index c1bc6b47..5b25739f 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbMoveTest.java @@ -1,52 +1,41 @@ package org.tron.plugins; -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; - import java.io.File; import java.io.IOException; import java.net.URL; import java.nio.file.Paths; -import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; +import org.junit.After; import org.junit.Assert; -import org.junit.BeforeClass; +import org.junit.Rule; import org.junit.Test; -import org.tron.plugins.utils.FileUtils; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbMoveTest { private static final String OUTPUT_DIRECTORY = "output-directory-toolkit"; - private static final String OUTPUT_DIRECTORY_DATABASE = - Paths.get(OUTPUT_DIRECTORY,"ori","database").toString(); - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ACCOUNT = "account"; - private static final String TRANS = "trans"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY_DATABASE, ACCOUNT); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private static final String ACCOUNT = "account"; + private static final String TRANS = "trans"; - file = new File(OUTPUT_DIRECTORY_DATABASE, MARKET); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); - file = new File(OUTPUT_DIRECTORY_DATABASE, TRANS); - factory.open(file, ArchiveManifest.newDefaultLevelDbOptions()).close(); - FileUtils.writeProperty(file + File.separator + ENGINE_FILE, ENGINE, LEVELDB); + private void init(DbTool.DbType dbType, String path) throws IOException, RocksDBException { + DbTool.getDB(path, ACCOUNT, dbType).close(); + DbTool.getDB(path, DBUtils.MARKET_PAIR_PRICE_TO_ORDER, dbType).close(); + DbTool.getDB(path, TRANS, dbType).close(); } - @AfterClass - public static void destroy() { + @After + public void destroy() { deleteDir(new File(OUTPUT_DIRECTORY)); } @@ -74,9 +63,23 @@ private static String getConfig(String config) { } @Test - public void testMv() { + public void testMvForLevelDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.LevelDB, Paths.get(database.getPath()).toString()); + String[] args = new String[] {"db", "mv", "-d", + database.getParent(), "-c", + getConfig("config.conf")}; + CommandLine cli = new CommandLine(new Toolkit()); + Assert.assertEquals(0, cli.execute(args)); + Assert.assertEquals(2, cli.execute(args)); + } + + @Test + public void testMvForRocksDB() throws RocksDBException, IOException { + File database = temporaryFolder.newFolder("database"); + init(DbTool.DbType.RocksDB, Paths.get(database.getPath()).toString()); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + database.getParent(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(0, cli.execute(args)); @@ -84,9 +87,10 @@ public void testMv() { } @Test - public void testDuplicate() { + public void testDuplicate() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", getConfig("config-duplicate.conf")}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); @@ -107,20 +111,19 @@ public void testDicNotExist() { } @Test - public void testConfNotExist() { + public void testConfNotExist() throws IOException { + File output = temporaryFolder.newFolder(); String[] args = new String[] {"db", "mv", "-d", - Paths.get(OUTPUT_DIRECTORY,"ori").toString(), "-c", + output.getPath(), "-c", "config.conf"}; CommandLine cli = new CommandLine(new Toolkit()); Assert.assertEquals(2, cli.execute(args)); } @Test - public void testEmpty() { - File file = new File(OUTPUT_DIRECTORY_DATABASE + File.separator + UUID.randomUUID()); - file.mkdirs(); - file.deleteOnExit(); - String[] args = new String[] {"db", "mv", "-d", file.toString(), "-c", + public void testEmpty() throws IOException { + File output = temporaryFolder.newFolder(); + String[] args = new String[] {"db", "mv", "-d", output.getPath(), "-c", getConfig("config.conf")}; CommandLine cli = new CommandLine(new Toolkit()); diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java index b86688f7..9d032a43 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbRootTest.java @@ -9,10 +9,8 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; import org.rocksdb.RocksDBException; -import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.db.DBInterface; import org.tron.plugins.utils.db.DbTool; -import org.tron.plugins.utils.db.LevelDBImpl; import picocli.CommandLine; @Slf4j @@ -27,8 +25,18 @@ public class DbRootTest { private static final String EMPTY_DB = "empty"; private static final String ERROR_DB = "error"; + + @Test + public void testRootForLevelDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.LevelDB); + } + @Test - public void testRoot() throws IOException, RocksDBException { + public void testRootForRocksDB() throws RocksDBException, IOException { + testRoot(DbTool.DbType.RocksDB); + } + + public void testRoot(DbTool.DbType dbType) throws IOException, RocksDBException { File file = folder.newFolder(); @@ -36,8 +44,8 @@ public void testRoot() throws IOException, RocksDBException { Assert.assertTrue(database.mkdirs()); - try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, DbTool.DbType.LevelDB); - DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, DbTool.DbType.RocksDB)) { + try (DBInterface normal = DbTool.getDB(database.toString(), NORMAL_DB, dbType); + DBInterface empty = DbTool.getDB(database.toString(), EMPTY_DB, dbType)) { for (int i = 0; i < 10; i++) { normal.put(("" + i).getBytes(), (NORMAL_DB + "-" + i).getBytes()); } @@ -53,8 +61,7 @@ public void testRoot() throws IOException, RocksDBException { "--db", EMPTY_DB}; Assert.assertEquals(0, cli.execute(args)); - try (DBInterface errorDb = new LevelDBImpl( - DBUtils.newLevelDb(Paths.get(database.toString(), ERROR_DB)), ERROR_DB)) { + try (DBInterface errorDb = DbTool.getDB(database.toString(), ERROR_DB, dbType)) { for (int i = 0; i < 10; i++) { errorDb.put(("" + i).getBytes(), (ERROR_DB + "-" + i).getBytes()); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java index 8605fa18..bbcc1a0b 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/DbTest.java @@ -5,42 +5,43 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Paths; import java.util.UUID; -import org.iq80.leveldb.DB; -import org.junit.Before; +import org.junit.Assert; import org.junit.Rule; import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; import org.tron.plugins.utils.ByteArray; import org.tron.plugins.utils.DBUtils; import org.tron.plugins.utils.MarketUtils; +import org.tron.plugins.utils.db.DBInterface; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbTest { - String INPUT_DIRECTORY; + public String INPUT_DIRECTORY; private static final String ACCOUNT = "account"; private static final String MARKET = DBUtils.MARKET_PAIR_PRICE_TO_ORDER; - CommandLine cli = new CommandLine(new Toolkit()); - String tmpDir = System.getProperty("java.io.tmpdir"); + public CommandLine cli = new CommandLine(new Toolkit()); @Rule public final TemporaryFolder temporaryFolder = new TemporaryFolder(); - @Before - public void init() throws IOException { + public void init(DbTool.DbType dbType) throws IOException, RocksDBException { INPUT_DIRECTORY = temporaryFolder.newFolder().toString(); - initDB(new File(INPUT_DIRECTORY, ACCOUNT)); - initDB(new File(INPUT_DIRECTORY, MARKET)); - initDB(new File(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2)); + initDB(INPUT_DIRECTORY, ACCOUNT, dbType); + initDB(INPUT_DIRECTORY, MARKET, dbType); + initDB(INPUT_DIRECTORY, DBUtils.CHECKPOINT_DB_V2, dbType); } - private static void initDB(File file) throws IOException { - if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(file.getName())) { - File dbFile = new File(file, DBUtils.CHECKPOINT_DB_V2); + private static void initDB(String sourceDir, String dbName, DbTool.DbType dbType) + throws IOException, RocksDBException { + if (DBUtils.CHECKPOINT_DB_V2.equalsIgnoreCase(dbName)) { + File dbFile = new File(sourceDir, DBUtils.CHECKPOINT_DB_V2); if (dbFile.mkdirs()) { for (int i = 0; i < 3; i++) { - try (DB db = DBUtils.newLevelDb(Paths.get(dbFile.getPath(), - System.currentTimeMillis() + ""))) { + try (DBInterface db = DbTool.getDB(dbFile.getPath(), + System.currentTimeMillis() + "", dbType)) { for (int j = 0; j < 100; j++) { byte[] bytes = UUID.randomUUID().toString().getBytes(); db.put(bytes, bytes); @@ -50,8 +51,8 @@ private static void initDB(File file) throws IOException { } return; } - try (DB db = DBUtils.newLevelDb(file.toPath())) { - if (MARKET.equalsIgnoreCase(file.getName())) { + try (DBInterface db = DbTool.getDB(sourceDir, dbName, dbType)) { + if (MARKET.equalsIgnoreCase(dbName)) { byte[] sellTokenID1 = ByteArray.fromString("100"); byte[] buyTokenID1 = ByteArray.fromString("200"); byte[] pairPriceKey1 = MarketUtils.createPairPriceKey( @@ -73,26 +74,29 @@ private static void initDB(File file) throws IOException { 2003L ); - //Use out-of-order insertion，key in store should be 1,2,3 db.put(pairPriceKey1, "1".getBytes(StandardCharsets.UTF_8)); db.put(pairPriceKey2, "2".getBytes(StandardCharsets.UTF_8)); db.put(pairPriceKey3, "3".getBytes(StandardCharsets.UTF_8)); + Assert.assertEquals(3, db.size()); } else { for (int i = 0; i < 100; i++) { byte[] bytes = UUID.randomUUID().toString().getBytes(); db.put(bytes, bytes); } + Assert.assertEquals(100, db.size()); } } } /** * Generate a not-exist temporary directory path. + * * @return temporary path */ - public String genarateTmpDir() { - File dir = Paths.get(tmpDir, UUID.randomUUID().toString()).toFile(); + public String generateTmpDir() throws IOException { + File dir = Paths.get(temporaryFolder.newFolder().toString(), UUID.randomUUID().toString()) + .toFile(); dir.deleteOnExit(); return dir.getPath(); } diff --git a/tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java similarity index 69% rename from tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java index 4fd9e537..f5880d82 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/ArchiveManifestTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/ArchiveManifestTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -14,47 +12,41 @@ import java.nio.charset.StandardCharsets; import java.util.Properties; import java.util.UUID; - import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.ArchiveManifest; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; @Slf4j public class ArchiveManifestTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/archiveManifest"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; - @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); - - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java similarity index 71% rename from tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java index 10bed418..69dca01e 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbArchiveTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbArchiveTest.java @@ -1,6 +1,4 @@ -package org.tron.plugins; - -import static org.iq80.leveldb.impl.Iq80DBFactory.factory; +package org.tron.plugins.leveldb; import java.io.BufferedReader; import java.io.BufferedWriter; @@ -12,48 +10,43 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.nio.charset.StandardCharsets; -import java.nio.file.Paths; import java.util.Properties; import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.junit.AfterClass; import org.junit.Assert; import org.junit.BeforeClass; +import org.junit.ClassRule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.rocksdb.RocksDBException; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.DBUtils; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; @Slf4j public class DbArchiveTest { - private static final String OUTPUT_DIRECTORY = "output-directory/database/dbArchive"; + @ClassRule + public static final TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static String OUTPUT_DIRECTORY; - private static final String ENGINE = "ENGINE"; - private static final String LEVELDB = "LEVELDB"; - private static final String ROCKSDB = "ROCKSDB"; private static final String ACCOUNT = "account"; private static final String ACCOUNT_ROCKSDB = "account-rocksdb"; - private static final String MARKET = "market_pair_price_to_order"; - private static final String ENGINE_FILE = "engine.properties"; @BeforeClass - public static void init() throws IOException { - File file = new File(OUTPUT_DIRECTORY,ACCOUNT); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + public static void init() throws IOException, RocksDBException { + OUTPUT_DIRECTORY = temporaryFolder.newFolder("database").toString(); + File file = new File(OUTPUT_DIRECTORY, ACCOUNT); + DbTool.openLevelDb(file.toPath(),ACCOUNT).close(); - file = new File(OUTPUT_DIRECTORY,MARKET); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,LEVELDB); + file = new File(OUTPUT_DIRECTORY, DBUtils.MARKET_PAIR_PRICE_TO_ORDER); + DbTool.openLevelDb(file.toPath(), DBUtils.MARKET_PAIR_PRICE_TO_ORDER).close(); - file = new File(OUTPUT_DIRECTORY,ACCOUNT_ROCKSDB); - factory.open(file,ArchiveManifest.newDefaultLevelDbOptions()).close(); - writeProperty(file.toString() + File.separator + ENGINE_FILE,ENGINE,ROCKSDB); - - } + file = new File(OUTPUT_DIRECTORY, ACCOUNT_ROCKSDB); + DbTool.openRocksDb(file.toPath(), ACCOUNT_ROCKSDB).close(); - @AfterClass - public static void destroy() { - deleteDir(new File(OUTPUT_DIRECTORY)); } @Test diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java index 150e47c9..d24604f0 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbConvertTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbConvertTest.java @@ -1,27 +1,25 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import java.util.UUID; import org.junit.Assert; import org.junit.Test; +import org.rocksdb.RocksDBException; +import org.tron.plugins.DbTest; +import org.tron.plugins.Toolkit; +import org.tron.plugins.utils.db.DbTool; import picocli.CommandLine; public class DbConvertTest extends DbTest { @Test - public void testRun() throws IOException { + public void testRun() throws IOException, RocksDBException { + init(DbTool.DbType.LevelDB); String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, temporaryFolder.newFolder().toString() }; Assert.assertEquals(0, cli.execute(args)); } - @Test - public void testRunWithSafe() throws IOException { - String[] args = new String[] { "db", "convert", INPUT_DIRECTORY, - temporaryFolder.newFolder().toString(),"--safe" }; - Assert.assertEquals(0, cli.execute(args)); - } - @Test public void testHelp() { String[] args = new String[] {"db", "convert", "-h"}; diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java index 792f93ad..7666806e 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbTest extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java rename to tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java index ae48e1d6..de32ae29 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteLevelDbV2Test.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/leveldb/DbLiteLevelDbV2Test.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.leveldb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteLevelDbV2Test extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java similarity index 76% rename from tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java rename to tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java index e6910b11..2f9c92f9 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/DbLiteRocksDbTest.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbTest.java @@ -1,7 +1,8 @@ -package org.tron.plugins; +package org.tron.plugins.rocksdb; import java.io.IOException; import org.junit.Test; +import org.tron.plugins.DbLiteTest; public class DbLiteRocksDbTest extends DbLiteTest { diff --git a/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java new file mode 100644 index 00000000..ab1067fe --- /dev/null +++ b/tools/toolkit/src/test/java/org/tron/plugins/rocksdb/DbLiteRocksDbV2Test.java @@ -0,0 +1,13 @@ +package org.tron.plugins.rocksdb; + +import java.io.IOException; +import org.junit.Test; +import org.tron.plugins.DbLiteTest; + +public class DbLiteRocksDbV2Test extends DbLiteTest { + + @Test + public void testToolsWithRocksDB() throws InterruptedException, IOException { + testTools("ROCKSDB", 2); + } +} diff --git a/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java b/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java index ce191c83..5e21a693 100644 --- a/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java +++ b/tools/toolkit/src/test/java/org/tron/plugins/utils/PublicMethod.java @@ -6,22 +6,28 @@ import java.net.InetAddress; import java.net.Socket; import java.util.List; +import java.util.Objects; import java.util.Random; +import org.bouncycastle.util.encoders.Hex; import org.tron.api.GrpcAPI; import org.tron.api.WalletGrpc; import org.tron.common.crypto.ECKey; import org.tron.common.crypto.ECKey.ECDSASignature; import org.tron.common.parameter.CommonParameter; import org.tron.common.utils.Sha256Hash; +import org.tron.common.utils.Utils; import org.tron.core.Wallet; -import org.tron.protos.Protocol; import org.tron.protos.Protocol.Transaction; import org.tron.protos.Protocol.Transaction.Contract; -import org.tron.protos.contract.BalanceContract; import org.tron.protos.contract.BalanceContract.TransferContract; public class PublicMethod { + public static String getRandomPrivateKey() { + return Hex.toHexString(Objects + .requireNonNull(new ECKey(Utils.getRandom()).getPrivKeyBytes())); + } + /** * Transfer TRX. * @param to addr receives the asset @@ -47,15 +53,15 @@ public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priK while (times++ <= 2) { TransferContract.Builder builder = - BalanceContract.TransferContract.newBuilder(); - com.google.protobuf.ByteString bsTo = com.google.protobuf.ByteString.copyFrom(to); - com.google.protobuf.ByteString bsOwner = ByteString.copyFrom(owner); + TransferContract.newBuilder(); + ByteString bsTo = ByteString.copyFrom(to); + ByteString bsOwner = ByteString.copyFrom(owner); builder.setToAddress(bsTo); builder.setOwnerAddress(bsOwner); builder.setAmount(amount); TransferContract contract = builder.build(); - Protocol.Transaction transaction = blockingStubFull.createTransaction(contract); + Transaction transaction = blockingStubFull.createTransaction(contract); if (transaction == null || transaction.getRawData().getContractCount() == 0) { continue; } @@ -71,8 +77,8 @@ public static Boolean sendcoin(byte[] to, long amount, byte[] owner, String priK * @param ecKey ecKey of the private key * @param transaction transaction object */ - public static Protocol.Transaction signTransaction(ECKey ecKey, - Protocol.Transaction transaction) { + public static Transaction signTransaction(ECKey ecKey, + Transaction transaction) { if (ecKey == null || ecKey.getPrivKey() == null) { return null; } @@ -83,7 +89,7 @@ public static Protocol.Transaction signTransaction(ECKey ecKey, public static Transaction setTimestamp(Transaction transaction) { long currentTime = System.currentTimeMillis();//*1000000 + System.nanoTime()%1000000; Transaction.Builder builder = transaction.toBuilder(); - org.tron.protos.Protocol.Transaction.raw.Builder rowBuilder = transaction.getRawData() + Transaction.raw.Builder rowBuilder = transaction.getRawData() .toBuilder(); rowBuilder.setTimestamp(currentTime); builder.setRawData(rowBuilder.build()); @@ -113,7 +119,7 @@ public static Transaction sign(Transaction transaction, ECKey myKey) { * @param blockingStubFull Grpc interface */ public static GrpcAPI.Return broadcastTransaction( - Protocol.Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { + Transaction transaction, WalletGrpc.WalletBlockingStub blockingStubFull) { int i = 10; GrpcAPI.Return response = blockingStubFull.broadcastTransaction(transaction); while (!response.getResult() && response.getCode() == GrpcAPI.Return.response_code.SERVER_BUSY diff --git a/tools/toolkit/src/test/resources/config-localtest.conf b/tools/toolkit/src/test/resources/config-localtest.conf index b280997c..8049ceb6 100644 --- a/tools/toolkit/src/test/resources/config-localtest.conf +++ b/tools/toolkit/src/test/resources/config-localtest.conf @@ -90,7 +90,7 @@ node { minParticipationRate = 0 - fullNodeAllowShieldedTransaction = true + allowShieldedTransactionApi = true zenTokenId = 1000001 @@ -117,14 +117,21 @@ node { ] http { + fullNodeEnable = false fullNodePort = 8090 + solidityEnable = false solidityPort = 8091 + PBFTEnable = false + PBFTPort = 8092 } rpc { + enable = false port = 50051 - # default value is 50061 - # solidityPort = 50061 + solidityEnable = false + solidityPort = 50061 + PBFTEnable = false + PBFTPort = 50071 # Number of gRPC thread, default availableProcessors / 2 # thread = 16 @@ -158,6 +165,9 @@ node { # httpSolidityPort = 8555 # httpPBFTEnable = true # httpPBFTPort = 8565 + # maxBlockRange = 5000 + # maxSubTopics = 1000 + # maxBlockFilterNum = 30000 } } @@ -209,7 +219,7 @@ genesis.block = { witnesses = [ { - address: "TN3zfjYUmMFK3ZsHSsrdJoNRtGkQmZLBLz" + address: TEDapYSVvAZ3aYH7w8N9tMEEFKaNKUD5Bp url = "http://Test.org", voteCount = 106 }, @@ -246,7 +256,7 @@ genesis.block = { //localWitnessAccountAddress = TN3zfjYUmMFK3ZsHSsrdJoNRtGkQmZLBLz localwitness = [ - f4df789d3210ac881cb900464dd30409453044d2777060a0c391cbdf4c6a4f57 + 1234567890123456789012345678901234567890123456789012345678901234 ] diff --git a/tools/toolkit/src/test/resources/config.conf b/tools/toolkit/src/test/resources/config.conf index a596d68e..df7b703a 100644 --- a/tools/toolkit/src/test/resources/config.conf +++ b/tools/toolkit/src/test/resources/config.conf @@ -1,11 +1,5 @@ storage { - # Directory for storing persistent data - db.engine = "LEVELDB", - db.sync = false, - db.directory = "database", - index.directory = "index", - transHistory.switch = "on", properties = [ { name = "account",