Examples with RollingFileReader org.hyperledger.besu.util.io.RollingFileReader used on opensource projects

Example 1 with RollingFileReader

use of org.hyperledger.besu.util.io.RollingFileReader in project besu by hyperledger.

the class RestoreState method restoreBlocks.

private void restoreBlocks() throws IOException {
    try (final RollingFileReader headerReader = new RollingFileReader(this::headerFileName, compressed);
        final RollingFileReader bodyReader = new RollingFileReader(this::bodyFileName, compressed);
        final RollingFileReader receiptReader = new RollingFileReader(this::receiptFileName, compressed)) {
        final MutableBlockchain blockchain = besuController.getProtocolContext().getBlockchain();
        // target block is "including" the target block, so LE test not LT.
        for (long i = 0; i <= targetBlock; i++) {
            if (i % 100000 == 0) {
                LOG.info("Loading chain data {} / {}", i, targetBlock);
            }
            final byte[] headerEntry = headerReader.readBytes();
            final byte[] bodyEntry = bodyReader.readBytes();
            final byte[] receiptEntry = receiptReader.readBytes();
            final BlockHeaderFunctions functions = new MainnetBlockHeaderFunctions();
            final BlockHeader header = BlockHeader.readFrom(new BytesValueRLPInput(Bytes.wrap(headerEntry), false, true), functions);
            final BlockBody body = BlockBody.readFrom(new BytesValueRLPInput(Bytes.wrap(bodyEntry), false, true), functions);
            final RLPInput receiptsRlp = new BytesValueRLPInput(Bytes.wrap(receiptEntry), false, true);
            final int receiptsCount = receiptsRlp.enterList();
            final List<TransactionReceipt> receipts = new ArrayList<>(receiptsCount);
            for (int j = 0; j < receiptsCount; j++) {
                receipts.add(TransactionReceipt.readFrom(receiptsRlp, true));
            }
            receiptsRlp.leaveList();
            blockchain.appendBlock(new Block(header, body), receipts);
        }
    }
    LOG.info("Chain data loaded");
}

Example 2 with RollingFileReader

use of org.hyperledger.besu.util.io.RollingFileReader in project besu by hyperledger.

the class RollingImport method main.

public static void main(final String[] arg) throws IOException {
    checkArgument(arg.length == 1, "Single argument is file prefix, like `./layer/besu-layer`");
    final RollingFileReader reader = new RollingFileReader((i, c) -> Path.of(String.format(arg[0] + "-%04d.rdat", i)), false);
    final InMemoryKeyValueStorageProvider provider = new InMemoryKeyValueStorageProvider();
    final BonsaiWorldStateArchive archive = new BonsaiWorldStateArchive(provider, null);
    final InMemoryKeyValueStorage accountStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_INFO_STATE);
    final InMemoryKeyValueStorage codeStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.CODE_STORAGE);
    final InMemoryKeyValueStorage storageStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.ACCOUNT_STORAGE_STORAGE);
    final InMemoryKeyValueStorage trieBranchStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_BRANCH_STORAGE);
    final InMemoryKeyValueStorage trieLogStorage = (InMemoryKeyValueStorage) provider.getStorageBySegmentIdentifier(KeyValueSegmentIdentifier.TRIE_LOG_STORAGE);
    final BonsaiPersistedWorldState bonsaiState = new BonsaiPersistedWorldState(archive, new BonsaiWorldStateKeyValueStorage(accountStorage, codeStorage, storageStorage, trieBranchStorage, trieLogStorage));
    int count = 0;
    while (!reader.isDone()) {
        try {
            final byte[] bytes = reader.readBytes();
            if (bytes.length < 1) {
                continue;
            }
            final TrieLogLayer layer = TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false));
            final BonsaiWorldStateUpdater updater = (BonsaiWorldStateUpdater) bonsaiState.updater();
            updater.rollForward(layer);
            updater.commit();
            bonsaiState.persist(null);
            if (count % 10000 == 0) {
                System.out.println(". - " + count);
            } else if (count % 100 == 0) {
                System.out.print(".");
                System.out.flush();
            }
        } catch (final Exception e) {
            // e.printStackTrace(System.out);
            System.out.println(count);
            throw e;
        }
        count++;
    }
    System.out.printf("%nCount %d - now going backwards!%n", count);
    while (count > 0) {
        try {
            count--;
            reader.seek(count);
            final byte[] bytes = reader.readBytes();
            final TrieLogLayer layer = TrieLogLayer.readFrom(new BytesValueRLPInput(Bytes.wrap(bytes), false));
            final BonsaiWorldStateUpdater updater = (BonsaiWorldStateUpdater) bonsaiState.updater();
            updater.rollBack(layer);
            updater.commit();
            bonsaiState.persist(null);
            if (count % 10000 == 0) {
                System.out.println(". - " + count);
            } else if (count % 100 == 0) {
                System.out.print(".");
                System.out.flush();
            }
        } catch (final Exception e) {
            System.out.println(count);
            throw e;
        }
    }
    System.out.printf("Back to zero!%n");
    accountStorage.dump(System.out);
    codeStorage.dump(System.out);
    storageStorage.dump(System.out);
    trieBranchStorage.dump(System.out);
    trieLogStorage.dump(System.out);
}

Example 3 with RollingFileReader

use of org.hyperledger.besu.util.io.RollingFileReader in project besu by hyperledger.

the class RestoreState method restoreAccounts.

@SuppressWarnings("UnusedVariable")
private void restoreAccounts() throws IOException {
    newWorldStateUpdater();
    int storageBranchCount = 0;
    int storageExtensionCount = 0;
    int storageLeafCount = 0;
    final PersistVisitor<Bytes> accountPersistVisitor = new PersistVisitor<>(this::updateAccountState);
    Node<Bytes> root = accountPersistVisitor.initialRoot();
    try (final RollingFileReader reader = new RollingFileReader(this::accountFileName, compressed)) {
        for (long i = 0; i < accountCount; i++) {
            if (i % 100000 == 0) {
                LOG.info("Loading account data {} / {}", i, accountCount);
            }
            final byte[] accountEntry = reader.readBytes();
            final BytesValueRLPInput accountInput = new BytesValueRLPInput(Bytes.of(accountEntry), false, true);
            final int length = accountInput.enterList();
            if (length != 3) {
                throw new RuntimeException("Unexpected account length " + length);
            }
            final Bytes32 trieKey = accountInput.readBytes32();
            final Bytes accountRlp = accountInput.readBytes();
            final Bytes code = accountInput.readBytes();
            final StateTrieAccountValue trieAccount = StateTrieAccountValue.readFrom(new BytesValueRLPInput(accountRlp, false, true));
            if (!trieAccount.getCodeHash().equals(Hash.hash(code))) {
                throw new RuntimeException("Code hash doesn't match");
            }
            if (code.size() > 0) {
                updateCode(code);
            }
            final RestoreVisitor<Bytes> accountTrieWriteVisitor = new RestoreVisitor<>(t -> t, accountRlp, accountPersistVisitor);
            root = root.accept(accountTrieWriteVisitor, bytesToPath(trieKey));
            final PersistVisitor<Bytes> storagePersistVisitor = new PersistVisitor<>(this::updateAccountStorage);
            Node<Bytes> storageRoot = storagePersistVisitor.initialRoot();
            while (true) {
                final byte[] trieEntry = reader.readBytes();
                final BytesValueRLPInput trieInput = new BytesValueRLPInput(Bytes.of(trieEntry), false, true);
                final int len = trieInput.enterList();
                if (len == 0) {
                    break;
                }
                if (len != 2) {
                    throw new RuntimeException("Unexpected storage trie entry length " + len);
                }
                final Bytes32 storageTrieKey = Bytes32.wrap(trieInput.readBytes());
                final Bytes storageTrieValue = Bytes.wrap(trieInput.readBytes());
                final RestoreVisitor<Bytes> storageTrieWriteVisitor = new RestoreVisitor<>(t -> t, storageTrieValue, storagePersistVisitor);
                storageRoot = storageRoot.accept(storageTrieWriteVisitor, bytesToPath(storageTrieKey));
                trieInput.leaveList();
            }
            storagePersistVisitor.persist(storageRoot);
            storageBranchCount += storagePersistVisitor.getBranchNodeCount();
            storageExtensionCount += storagePersistVisitor.getExtensionNodeCount();
            storageLeafCount += storagePersistVisitor.getLeafNodeCount();
            accountInput.leaveList();
        }
    }
    accountPersistVisitor.persist(root);
    updater.commit();
    LOG.info("Account BranchNodes: {} ", accountPersistVisitor.getBranchNodeCount());
    LOG.info("Account ExtensionNodes: {} ", accountPersistVisitor.getExtensionNodeCount());
    LOG.info("Account LeafNodes: {} ", accountPersistVisitor.getLeafNodeCount());
    LOG.info("Storage BranchNodes: {} ", storageBranchCount);
    LOG.info("Storage LeafNodes: {} ", storageExtensionCount);
    LOG.info("Storage ExtensionNodes: {} ", storageLeafCount);
    LOG.info("Account data loaded");
}