Examples with VirtualLeafRecordSerializer com.swirlds.jasperdb.VirtualLeafRecordSerializer used on opensource projects

Example 1 with VirtualLeafRecordSerializer

use of com.swirlds.jasperdb.VirtualLeafRecordSerializer in project hedera-services by hashgraph.

the class ContractBench method prepare.

@Setup
public void prepare() throws Exception {
    pipeline = new Pipeline<>();
    final long estimatedNumKeyValuePairs = (long) (numContracts * (1 - bigPercent - hugePercent) * ((kbPerContract * 1024L) / ESTIMATED_KEY_VALUE_SIZE)) + (long) (numContracts * bigPercent * ((kbPerBigContract * 1024L) / ESTIMATED_KEY_VALUE_SIZE)) + (long) (numContracts * hugePercent * ((kbPerHugeContract * 1024L) / ESTIMATED_KEY_VALUE_SIZE));
    System.out.println("estimatedNumKeyValuePairs = " + estimatedNumKeyValuePairs);
    VirtualLeafRecordSerializer<ContractKey, ContractValue> virtualLeafRecordSerializer = new VirtualLeafRecordSerializer<>((short) 1, DigestType.SHA_384, (short) 1, DataFileCommon.VARIABLE_DATA_SIZE, new ContractKeySupplier(), (short) 1, ContractValue.SERIALIZED_SIZE, new ContractValueSupplier(), true);
    Path dataSourcePath = getDataSourcePath(dsType);
    boolean dataSourceDirExisted = Files.exists(dataSourcePath);
    virtualMap = createMap(dsType, virtualLeafRecordSerializer, new ContractKeySerializer(), estimatedNumKeyValuePairs, dataSourcePath, preferDiskBasedIndexes);
    txProcessor = new TransactionProcessor<>(preFetchEventHandlers, (Transaction<Data> tx) -> {
        // preFetch logic
        VirtualMap<ContractKey, ContractValue> map = getVirtualMap();
        final Data data = tx.getData();
        data.value1 = map.getForModify(data.key1);
        data.value2 = map.getForModify(data.key2);
    }, (Transaction<Data> tx) -> {
        // handleTransaction logic
        final Data data = tx.getData();
        data.value1.setValue(data.value1.asLong() - data.transferAmount);
        data.value2.setValue(data.value2.asLong() + data.transferAmount);
    });
    // We generate a different number of key/value pairs depending on whether it is
    // a huge contract, big contract, or normal contract
    int numBigContracts = (int) (numContracts * bigPercent);
    System.out.println("numBigContracts = " + numBigContracts);
    int numHugeContracts = (int) (numContracts * hugePercent);
    System.out.println("numHugeContracts = " + numHugeContracts);
    keyValuePairsPerContract = new int[numContracts];
    for (int i = 0; i < numContracts; i++) {
        final int kb;
        if (i > 0 && (i % 100) == 0 && numHugeContracts > 0) {
            kb = kbPerHugeContract;
            numHugeContracts--;
        } else if (i > 0 && (i % 10) == 0 && numBigContracts > 0) {
            kb = kbPerBigContract;
            numBigContracts--;
        } else {
            kb = kbPerContract;
        }
        final var numKeyValuePairs = (kb * 1024L) / ESTIMATED_KEY_VALUE_SIZE;
        keyValuePairsPerContract[i] = (int) numKeyValuePairs;
    }
    if (!dataSourceDirExisted && preFill) {
        long countOfKeyValuePairs = 0;
        long lastCountOfKeyValuePairs = 0;
        for (int i = 0; i < numContracts; i++) {
            if ((countOfKeyValuePairs - lastCountOfKeyValuePairs) > 100_000) {
                lastCountOfKeyValuePairs = countOfKeyValuePairs;
                System.out.printf("Completed: %,d contracts and %,d key/value pairs\n", i, countOfKeyValuePairs);
                virtualMap = pipeline.endRound(virtualMap);
            }
            if (i > 0 && i % 10000 == 0) {
                System.out.println("=============== GC =======================");
                // loading is really intense so give GC a chance to catch up
                System.gc();
                Thread.sleep(1000);
            }
            final int numKeyValuePairs = keyValuePairsPerContract[i];
            for (int j = 0; j < numKeyValuePairs; j++) {
                final var key = asContractKey(i, j);
                final var value = new ContractValue(j);
                try {
                    virtualMap.put(key, value);
                } catch (Exception e) {
                    e.printStackTrace();
                    System.err.println(i + ":" + j);
                    throw e;
                }
            }
            countOfKeyValuePairs += numKeyValuePairs;
        }
        // During setup, we perform the full hashing and release the old copy. This way,
        // during the tests, we don't have an initial slow hash.
        System.out.printf("Completed: %,d contracts and %,d key/value pairs\n", numContracts, countOfKeyValuePairs);
        virtualMap = pipeline.endRound(virtualMap);
    } else {
        System.out.println("NOT PRE_FILLING AS LOADED FROM FILES OR TURNED OFF WITH FLAG!");
    }
    printDataStoreSize();
    // create a snapshot every 15min
    DateFormat df = new SimpleDateFormat("yyyy-MM-dd--HH-mm");
    ScheduledExecutorService snapshotting = Executors.newScheduledThreadPool(1, runnable -> new Thread(runnable, "Snapshot"));
    snapshotting.scheduleWithFixedDelay(() -> {
        final Path snapshotDir = Path.of("jasperdb_snapshot_" + df.format(new Date()));
        System.out.println("************ STARTING SNAPSHOT [" + snapshotDir.toAbsolutePath() + "] ***********");
        long START = System.currentTimeMillis();
        try {
            virtualMap.getDataSource().snapshot(snapshotDir);
        } catch (IOException e) {
            e.printStackTrace();
        }
        double tookSeconds = (System.currentTimeMillis() - START) * Units.MILLISECONDS_TO_SECONDS;
        System.out.printf("************ SNAPSHOT FINISHED took %,3f seconds [%s] ***********\n", tookSeconds, snapshotDir.toAbsolutePath());
    }, 0, 5, TimeUnit.MINUTES);
}

Example 2 with VirtualLeafRecordSerializer

use of com.swirlds.jasperdb.VirtualLeafRecordSerializer in project hedera-services by hashgraph.

the class VirtualDataSourceBench method setup.

@Setup(Level.Trial)
public void setup() throws IOException {
    random = new Random(1234);
    System.out.println("dataSourcePath = " + dataSourcePath);
    if (Files.exists(dataSourcePath)) {
        System.err.println("!!!!!!!!!!!!! Deleting old db.");
        deleteDirectoryAndContents(dataSourcePath);
    }
    if (Files.exists(dataSourceSnapshotPath)) {
        System.err.println("!!!!!!!!!!!!! Deleting old db snapshot.");
        deleteDirectoryAndContents(dataSourceSnapshotPath);
    }
    // create data source
    VirtualLeafRecordSerializer<ContractKey, ContractValue> virtualLeafRecordSerializer = new VirtualLeafRecordSerializer<>((short) 1, DigestType.SHA_384, (short) 1, DataFileCommon.VARIABLE_DATA_SIZE, new ContractKeySupplier(), (short) 1, ContractValue.SERIALIZED_SIZE, new ContractValueSupplier(), true);
    JasperDbBuilder<ContractKey, ContractValue> dbBuilder = new JasperDbBuilder<>();
    dbBuilder.virtualLeafRecordSerializer(virtualLeafRecordSerializer).virtualInternalRecordSerializer(new VirtualInternalRecordSerializer()).keySerializer(new ContractKeySerializer()).storageDir(dataSourcePath).maxNumOfKeys(500_000_000).preferDiskBasedIndexes(false).internalHashesRamToDiskThreshold(0).mergingEnabled(false);
    dataSource = dbBuilder.build("jdb", "4dsBench");
    // populate with initial data
    System.out.printf("Creating initial data set of %,d leaves\n", initialDataSize);
    progressPercentage = 0;
    final long firstLeafPath = initialDataSize;
    final long lastLeafPath = firstLeafPath + initialDataSize;
    var internalRecordStream = LongStream.range(0, firstLeafPath).mapToObj(path -> new VirtualInternalRecord(path, hash((int) path)));
    var leafRecordStream = LongStream.range(firstLeafPath, lastLeafPath + 1).mapToObj(path -> new VirtualLeafRecord<>(path, hash((int) path), new ContractKey(path, path), new ContractValue(path))).peek(leaf -> printProgress(leaf.getPath(), lastLeafPath));
    dataSource.saveRecords(firstLeafPath, lastLeafPath, internalRecordStream, leafRecordStream, Stream.empty());
    System.out.printf("Done creating initial data set of %,d leaves\n", initialDataSize);
}

Example 3 with VirtualLeafRecordSerializer

use of com.swirlds.jasperdb.VirtualLeafRecordSerializer in project hedera-services by hashgraph.

the class ContractDataSourceValidator method main.

public static void main(String[] args) throws IOException {
    Path dataSourcePath = Path.of("").toAbsolutePath();
    System.out.println("dataSourcePath = " + dataSourcePath + " exists " + Files.exists(dataSourcePath));
    VirtualLeafRecordSerializer<ContractKey, ContractValue> virtualLeafRecordSerializer = new VirtualLeafRecordSerializer<>((short) 1, DigestType.SHA_384, (short) 1, DataFileCommon.VARIABLE_DATA_SIZE, new ContractKeySupplier(), (short) 1, ContractValue.SERIALIZED_SIZE, new ContractValueSupplier(), true);
    ;
    JasperDbBuilder<ContractKey, ContractValue> dbBuilder = new JasperDbBuilder<>();
    dbBuilder.virtualLeafRecordSerializer(virtualLeafRecordSerializer).virtualInternalRecordSerializer(new VirtualInternalRecordSerializer()).keySerializer(new ContractKeySerializer()).storageDir(dataSourcePath).maxNumOfKeys(500_000_000).preferDiskBasedIndexes(false).internalHashesRamToDiskThreshold(0).mergingEnabled(true);
    DataSourceValidator<ContractKey, ContractValue> dataSourceValidator = new DataSourceValidator<>(dbBuilder.build("jdb", "4validator"));
    dataSourceValidator.validate();
}

Example 4 with VirtualLeafRecordSerializer

use of com.swirlds.jasperdb.VirtualLeafRecordSerializer in project hedera-services by hashgraph.

the class CryptoHbarBench method prepare.

@Setup
public void prepare() throws Exception {
    pipeline = new Pipeline<>();
    VirtualLeafRecordSerializer<Id, Account> virtualLeafRecordSerializer = new VirtualLeafRecordSerializer<>((short) 1, DigestType.SHA_384, (short) 1, Id.SERIALIZED_SIZE, new IdSupplier(), (short) 1, Account.SERIALIZED_SIZE, new AccountSupplier(), false);
    Path dataSourcePath = getDataSourcePath(dsType);
    boolean dataSourceDirExisted = Files.exists(dataSourcePath);
    virtualMap = createMap(dsType, virtualLeafRecordSerializer, new Id.IdKeySerializer(), numEntities, dataSourcePath, false);
    final var rand = new Random();
    txProcessor = new TransactionProcessor<>(preFetchEventHandlers, (Transaction<Data> tx) -> {
        // preFetch logic
        VirtualMap<Id, Account> map = getVirtualMap();
        final Data data = tx.getData();
        final Account sender = map.getForModify(data.getSenderId());
        if (sender == null) {
            System.out.println("NULL SENDER " + data.getSenderId() + ", last = " + tx.isLast());
        }
        final Account receiver = map.getForModify(data.getReceiverId());
        if (receiver == null)
            System.out.println("NULL RECEIVER " + data.getReceiverId() + ", last = " + tx.isLast());
        data.setSender(map.getForModify(data.getSenderId()));
        data.setReceiver(map.getForModify(data.getReceiverId()));
    }, (Transaction<Data> tx) -> {
        // handleTransaction logic
        final var tinyBars = rand.nextInt(10);
        final Data data = tx.getData();
        final Account sender = data.getSender();
        final Account receiver = data.getReceiver();
        sender.setHbarBalance(sender.getHbarBalance() - tinyBars);
        receiver.setHbarBalance(receiver.getHbarBalance() + tinyBars);
    });
    long START = System.currentTimeMillis();
    if (!dataSourceDirExisted && preFill) {
        for (int i = 0; i < numEntities; i++) {
            if (i % 100000 == 0 && i > 0) {
                final long END = System.currentTimeMillis();
                double tookSeconds = (END - START) / 1000d;
                START = END;
                System.out.printf("Completed: %,d in %,.2f seconds\n", i, tookSeconds);
                virtualMap = pipeline.endRound(virtualMap);
            }
            if (numEntities > 100_000_000) {
                // for large data loads give the GC a chance as we crate object like a crazy beast!
                if (i % 1_000_000 == 0 && i > 0) {
                    System.gc();
                    // noinspection BusyWait
                    Thread.sleep(2000);
                    System.gc();
                    // noinspection BusyWait
                    Thread.sleep(2000);
                }
            }
            final var key = asId(i);
            final var value = asAccount(i);
            try {
                virtualMap.put(key, value);
            } catch (Exception e) {
                e.printStackTrace();
                System.err.println(i);
                throw e;
            }
        }
        System.out.printf("Completed: %,d\n", numEntities);
        virtualMap = pipeline.endRound(virtualMap);
    }
    printDataStoreSize();
// prepNextRound();
}

Example 5 with VirtualLeafRecordSerializer

use of com.swirlds.jasperdb.VirtualLeafRecordSerializer in project hedera-services by hashgraph.

the class VirtualDataSourceNewAPIBench method setup.

@Setup(Level.Trial)
public void setup() {
    System.out.println("------- Setup -----------------------------");
    storePath = Path.of("store-" + impl);
    try {
        final boolean storeExists = Files.exists(storePath);
        // get slot index suppliers
        switch(impl) {
            case "lmdb":
                dataSource = new VFCDataSourceLmdb<>(// max seralized size
                1 + 8 + 32, // max seralized size
                ContractKey::new, ContractValue.SERIALIZED_SIZE, ContractValue::new, storePath);
                break;
            // storePath);
            case "jasperdb":
                VirtualLeafRecordSerializer<ContractKey, ContractValue> virtualLeafRecordSerializer = new VirtualLeafRecordSerializer<>((short) 1, DigestType.SHA_384, (short) 1, DataFileCommon.VARIABLE_DATA_SIZE, new ContractKeySupplier(), (short) 1, ContractValue.SERIALIZED_SIZE, new ContractValueSupplier(), true);
                JasperDbBuilder<ContractKey, ContractValue> dbBuilder = new JasperDbBuilder<>();
                dbBuilder.virtualLeafRecordSerializer(virtualLeafRecordSerializer).virtualInternalRecordSerializer(new VirtualInternalRecordSerializer()).keySerializer(new ContractKeySerializer()).storageDir(storePath).maxNumOfKeys(numEntities + 10_000_000).preferDiskBasedIndexes(false).internalHashesRamToDiskThreshold(Long.MAX_VALUE).mergingEnabled(true);
                dataSource = dbBuilder.build("jdb", "4ApiBench");
                break;
            default:
                throw new IllegalStateException("Unexpected value: " + impl);
        }
        ;
        // create data
        if (!storeExists) {
            System.out.println("================================================================================");
            System.out.println("Creating data ...");
            // create internal nodes and leaves
            long iHaveWritten = 0;
            while (iHaveWritten < numEntities) {
                final long start = System.currentTimeMillis();
                final long batchSize = Math.min(WRITE_BATCH_SIZE, numEntities - iHaveWritten);
                dataSource.saveRecords(numEntities, numEntities * 2, LongStream.range(iHaveWritten, iHaveWritten + batchSize).mapToObj(i -> new VirtualInternalRecord(i, hash((int) i))), LongStream.range(iHaveWritten, iHaveWritten + batchSize).mapToObj(i -> new VirtualLeafRecord<>(i + numEntities, hash((int) i), new ContractKey(i, i), new ContractValue(i))), Stream.empty());
                iHaveWritten += batchSize;
                printUpdate(start, batchSize, ContractValue.SERIALIZED_SIZE, "Created " + iHaveWritten + " Nodes");
            }
            System.out.println("================================================================================");
            // set nextPath
            nextPath = numEntities;
            // let merge catch up
            try {
                System.out.println("Waiting for merge");
                Thread.sleep(30000);
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
        } else {
            System.out.println("Loaded existing data");
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
}