Examples with StoredMerklePatriciaTrie org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie used on opensource projects

Example 1 with StoredMerklePatriciaTrie

use of org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie in project besu by hyperledger.

the class StateBackupService method visitAccount.

private TrieIterator.State visitAccount(final Bytes32 nodeKey, final Node<Bytes> node) {
    if (node.getValue().isEmpty()) {
        return State.CONTINUE;
    }
    backupStatus.currentAccount = nodeKey;
    final Bytes nodeValue = node.getValue().orElse(Hash.EMPTY);
    final StateTrieAccountValue account = StateTrieAccountValue.readFrom(new BytesValueRLPInput(nodeValue, false));
    final Bytes code = worldStateStorage.getCode(account.getCodeHash(), null).orElse(Bytes.EMPTY);
    backupStatus.codeSize.addAndGet(code.size());
    final BytesValueRLPOutput accountOutput = new BytesValueRLPOutput();
    accountOutput.startList();
    // trie hash
    accountOutput.writeBytes(nodeKey);
    // account rlp
    accountOutput.writeBytes(nodeValue);
    // code
    accountOutput.writeBytes(code);
    accountOutput.endList();
    try {
        accountFileWriter.writeBytes(accountOutput.encoded().toArrayUnsafe());
    } catch (final IOException ioe) {
        LOG.error("Failure writing backup", ioe);
        return State.STOP;
    }
    // storage is written for each leaf, otherwise the whole trie would have to fit in memory
    final StoredMerklePatriciaTrie<Bytes32, Bytes> storageTrie = new StoredMerklePatriciaTrie<>(worldStateStorage::getAccountStateTrieNode, account.getStorageRoot(), Function.identity(), Function.identity());
    storageTrie.visitLeafs((storageKey, storageValue) -> visitAccountStorage(storageKey, storageValue, accountFileWriter));
    try {
        accountFileWriter.writeBytes(ACCOUNT_END_MARKER.toArrayUnsafe());
    } catch (final IOException ioe) {
        LOG.error("Failure writing backup", ioe);
        return State.STOP;
    }
    backupStatus.accountCount.incrementAndGet();
    return State.CONTINUE;
}

Example 2 with StoredMerklePatriciaTrie

use of org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie in project besu by hyperledger.

the class FastWorldStateDownloaderTest method doesNotRequestKnownStorageTrieNodesFromNetwork.

@Test
public void doesNotRequestKnownStorageTrieNodesFromNetwork() {
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    final List<Account> accounts = dataGen.createRandomContractAccountsWithNonEmptyStorage(remoteWorldState, 20);
    final Hash stateRoot = remoteWorldState.rootHash();
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Create some peers
    final List<RespondingEthPeer> peers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(5).collect(Collectors.toList());
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    // Seed local storage with some trie node values
    final List<Bytes32> storageRootHashes = new StoredMerklePatriciaTrie<>(remoteStorage::getNodeData, remoteWorldState.rootHash(), Function.identity(), Function.identity()).entriesFrom(Bytes32.ZERO, 5).values().stream().map(RLP::input).map(StateTrieAccountValue::readFrom).map(StateTrieAccountValue::getStorageRoot).collect(Collectors.toList());
    final Map<Bytes32, Bytes> allTrieNodes = new HashMap<>();
    final Set<Bytes32> knownNodes = new HashSet<>();
    final Set<Bytes32> unknownNodes = new HashSet<>();
    for (final Bytes32 storageRootHash : storageRootHashes) {
        allTrieNodes.putAll(collectTrieNodesToBeRequestedAfterRoot(remoteStorage, storageRootHash, 5));
    }
    // Sanity check
    assertThat(allTrieNodes.size()).isGreaterThan(0);
    final Updater localStorageUpdater = localStorage.updater();
    boolean storeNode = true;
    for (final Map.Entry<Bytes32, Bytes> entry : allTrieNodes.entrySet()) {
        final Bytes32 hash = entry.getKey();
        final Bytes data = entry.getValue();
        if (storeNode) {
            localStorageUpdater.putAccountStorageTrieNode(null, null, hash, data);
            knownNodes.add(hash);
        } else {
            unknownNodes.add(hash);
        }
        storeNode = !storeNode;
    }
    localStorageUpdater.commit();
    final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection);
    final FastSyncState fastSyncState = new FastSyncState(header);
    final CompletableFuture<Void> result = downloader.run(null, fastSyncState);
    // Respond to node data requests
    final List<MessageData> sentMessages = new ArrayList<>();
    final RespondingEthPeer.Responder blockChainResponder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive, mock(TransactionPool.class));
    final RespondingEthPeer.Responder responder = RespondingEthPeer.wrapResponderWithCollector(blockChainResponder, sentMessages);
    respondUntilDone(peers, responder, result);
    // World state should be available by the time the result is complete
    assertThat(localStorage.isWorldStateAvailable(stateRoot, header.getHash())).isTrue();
    // Check that unknown trie nodes were requested
    final List<Bytes32> requestedHashes = sentMessages.stream().filter(m -> m.getCode() == EthPV63.GET_NODE_DATA).map(GetNodeDataMessage::readFrom).flatMap(m -> StreamSupport.stream(m.hashes().spliterator(), true)).collect(Collectors.toList());
    assertThat(requestedHashes.size()).isGreaterThan(0);
    assertThat(requestedHashes).containsAll(unknownNodes);
    assertThat(requestedHashes).doesNotContainAnyElementsOf(knownNodes);
    // Check that all expected account data was downloaded
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertThat(result).isDone();
    assertAccountsMatch(localWorldState, accounts);
}

Example 3 with StoredMerklePatriciaTrie

use of org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie in project besu by hyperledger.

the class PersistDataStepTest method assertDataPersisted.

private void assertDataPersisted(final List<Task<SnapDataRequest>> tasks) {
    tasks.forEach(task -> {
        if (task.getData() instanceof AccountRangeDataRequest) {
            final AccountRangeDataRequest data = (AccountRangeDataRequest) task.getData();
            StoredMerklePatriciaTrie<Bytes, Bytes> trie = new StoredMerklePatriciaTrie<>(worldStateStorage::getAccountStateTrieNode, data.getRootHash(), b -> b, b -> b);
            data.getAccounts().forEach((key, value) -> assertThat(trie.get(key)).isPresent());
        } else if (task.getData() instanceof StorageRangeDataRequest) {
            final StorageRangeDataRequest data = (StorageRangeDataRequest) task.getData();
            final StoredMerklePatriciaTrie<Bytes, Bytes> trie = new StoredMerklePatriciaTrie<>((location, hash) -> worldStateStorage.getAccountStorageTrieNode(Hash.wrap(data.getAccountHash()), location, hash), data.getStorageRoot(), b -> b, b -> b);
            data.getSlots().forEach((key, value) -> assertThat(trie.get(key)).isPresent());
        } else if (task.getData() instanceof BytecodeRequest) {
            final BytecodeRequest data = (BytecodeRequest) task.getData();
            assertThat(worldStateStorage.getCode(data.getCodeHash(), Hash.wrap(data.getAccountHash()))).isPresent();
        } else {
            fail("not expected message");
        }
    });
}

Example 4 with StoredMerklePatriciaTrie

use of org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie in project besu by hyperledger.

the class StackTrie method commit.

public void commit(final NodeUpdater nodeUpdater) {
    if (nbSegments.decrementAndGet() <= 0 && !elements.isEmpty()) {
        final List<Bytes> proofs = new ArrayList<>();
        final TreeMap<Bytes32, Bytes> keys = new TreeMap<>();
        elements.values().forEach(taskElement -> {
            proofs.addAll(taskElement.proofs());
            keys.putAll(taskElement.keys());
        });
        final Map<Bytes32, Bytes> proofsEntries = new HashMap<>();
        for (Bytes proof : proofs) {
            proofsEntries.put(Hash.hash(proof), proof);
        }
        final InnerNodeDiscoveryManager<Bytes> snapStoredNodeFactory = new InnerNodeDiscoveryManager<>((location, hash) -> Optional.ofNullable(proofsEntries.get(hash)), Function.identity(), Function.identity(), startKeyHash, keys.lastKey(), true);
        final MerklePatriciaTrie<Bytes, Bytes> trie = new StoredMerklePatriciaTrie<>(snapStoredNodeFactory, proofs.isEmpty() ? MerklePatriciaTrie.EMPTY_TRIE_NODE_HASH : rootHash);
        for (Map.Entry<Bytes32, Bytes> account : keys.entrySet()) {
            trie.put(account.getKey(), new SnapPutVisitor<>(snapStoredNodeFactory, account.getValue()));
        }
        trie.commit(nodeUpdater, (new CommitVisitor<>(nodeUpdater) {

            @Override
            public void maybeStoreNode(final Bytes location, final Node<Bytes> node) {
                if (!node.isHealNeeded()) {
                    super.maybeStoreNode(location, node);
                }
            }
        }));
    }
}

Example 5 with StoredMerklePatriciaTrie

use of org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie in project besu by hyperledger.

the class RangeManager method findNewBeginElementInRange.

/**
 * Helps to create a new range according to the last data obtained. This happens when a peer
 * doesn't return all of the data in a range.
 *
 * @param worldstateRootHash the root hash
 * @param proofs proof received
 * @param endKeyHash the end of the range initially wanted
 * @param receivedKeys the last key received
 * @return begin of the new range
 */
public static Optional<Bytes32> findNewBeginElementInRange(final Bytes32 worldstateRootHash, final List<Bytes> proofs, final TreeMap<Bytes32, Bytes> receivedKeys, final Bytes32 endKeyHash) {
    if (receivedKeys.isEmpty() || receivedKeys.lastKey().compareTo(endKeyHash) >= 0) {
        return Optional.empty();
    } else {
        final Map<Bytes32, Bytes> proofsEntries = new HashMap<>();
        for (Bytes proof : proofs) {
            proofsEntries.put(Hash.hash(proof), proof);
        }
        final StoredMerklePatriciaTrie<Bytes, Bytes> storageTrie = new StoredMerklePatriciaTrie<>(new InnerNodeDiscoveryManager<>((location, key) -> Optional.ofNullable(proofsEntries.get(key)), Function.identity(), Function.identity(), receivedKeys.lastKey(), endKeyHash, false), worldstateRootHash);
        try {
            storageTrie.visitAll(bytesNode -> {
            });
        } catch (MerkleTrieException e) {
            return Optional.of(InnerNodeDiscoveryManager.decodePath(e.getLocation()));
        }
        return Optional.empty();
    }
}