Examples with WorldStateKeyValueStorage org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage used on opensource projects

Example 1 with WorldStateKeyValueStorage

use of org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method testCancellation.

@SuppressWarnings("unchecked")
private void testCancellation(final boolean shouldCancelFuture) {
    final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create();
    // Prevent the persistence service from running
    final MockExecutorService serviceExecutor = ((DeterministicEthScheduler) ethProtocolManager.ethContext().getScheduler()).mockServiceExecutor();
    serviceExecutor.setAutoRun(false);
    // Setup "remote" state
    final WorldStateArchive remoteWorldStateArchive = createInMemoryWorldStateArchive();
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    dataGen.createRandomContractAccountsWithNonEmptyStorage(remoteWorldState, 20);
    final Hash stateRoot = remoteWorldState.rootHash();
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Create some peers
    final List<RespondingEthPeer> peers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(5).collect(Collectors.toList());
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection);
    final FastSyncState fastSyncState = new FastSyncState(header);
    final CompletableFuture<Void> result = downloader.run(null, fastSyncState);
    // Send a few responses
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    for (int i = 0; i < 3; i++) {
        for (final RespondingEthPeer peer : peers) {
            peer.respond(responder);
        }
        giveOtherThreadsAGo();
    }
    // Sanity check
    assertThat(result.isDone()).isFalse();
    // Reset taskCollection so we can track interactions after the cancellation
    reset(taskCollection);
    if (shouldCancelFuture) {
        result.cancel(true);
    } else {
        downloader.cancel();
        assertThat(result).isCancelled();
    }
    // Send some more responses after cancelling
    for (int i = 0; i < 3; i++) {
        for (final RespondingEthPeer peer : peers) {
            peer.respond(responder);
        }
        giveOtherThreadsAGo();
    }
    // Now allow the persistence service to run which should exit immediately
    serviceExecutor.runPendingFutures();
    verify(taskCollection, times(1)).clear();
    verify(taskCollection, never()).remove();
    verify(taskCollection, never()).add(any(NodeDataRequest.class));
    // Target world state should not be available
    assertThat(localStorage.isWorldStateAvailable(header.getStateRoot(), header.getHash())).isFalse();
}

Example 2 with WorldStateKeyValueStorage

use of org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method downloadAvailableWorldStateFromPeers.

private void downloadAvailableWorldStateFromPeers(final int peerCount, final int accountCount, final int hashesPerRequest, final int maxOutstandingRequests, final NetworkResponder networkResponder) {
    final int trailingPeerCount = 5;
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    final List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, accountCount);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Generate more data that should not be downloaded
    final List<Account> otherAccounts = dataGen.createRandomAccounts(remoteWorldState, 5);
    final Hash otherStateRoot = remoteWorldState.rootHash();
    final BlockHeader otherHeader = dataGen.block(BlockOptions.create().setStateRoot(otherStateRoot).setBlockNumber(11)).getHeader();
    // Sanity check
    assertThat(otherStateRoot).isNotEqualTo(stateRoot);
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateHashCountPerRequest(hashesPerRequest).worldStateRequestParallelism(maxOutstandingRequests).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create some peers that can respond
    final List<RespondingEthPeer> usefulPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(peerCount).collect(Collectors.toList());
    // And some irrelevant peers
    final List<RespondingEthPeer> trailingPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber() - 1L)).limit(trailingPeerCount).collect(Collectors.toList());
    // Start downloader
    final CompletableFuture<?> result = downloader.run(null, new FastSyncState(header));
    // A second run should return an error without impacting the first result
    final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
    assertThat(secondResult).isCompletedExceptionally();
    assertThat(result).isNotCompletedExceptionally();
    // Respond to node data requests
    // Send one round of full responses, so that we can get multiple requests queued up
    final RespondingEthPeer.Responder fullResponder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    for (final RespondingEthPeer peer : usefulPeers) {
        peer.respond(fullResponder);
    }
    // Respond to remaining queued requests in custom way
    networkResponder.respond(usefulPeers, remoteWorldStateArchive, result);
    // Check that trailing peers were not queried for data
    for (final RespondingEthPeer trailingPeer : trailingPeers) {
        assertThat(trailingPeer.hasOutstandingRequests()).isFalse();
    }
    // Check that all expected account data was downloaded
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertThat(result).isDone();
    assertAccountsMatch(localWorldState, accounts);
    // We shouldn't have any extra data locally
    assertThat(localStorage.contains(otherHeader.getStateRoot())).isFalse();
    for (final Account otherAccount : otherAccounts) {
        assertThat(localWorldState.get(otherAccount.getAddress())).isNull();
    }
}

Example 3 with WorldStateKeyValueStorage

use of org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method stalledDownloader.

@Test
public void stalledDownloader() {
    final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(new EthScheduler(1, 1, 1, 1, new NoOpMetricsSystem()));
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    dataGen.createRandomAccounts(remoteWorldState, 10);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create a peer that can respond
    final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber());
    // Start downloader (with a state root that's not available anywhere
    final CompletableFuture<Void> result = downloader.run(null, new FastSyncState(new BlockHeaderTestFixture().stateRoot(Hash.hash(Bytes.of(1, 2, 3, 4))).buildHeader()));
    // A second run should return an error without impacting the first result
    final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
    assertThat(secondResult).isCompletedExceptionally();
    assertThat(result).isNotCompletedExceptionally();
    final RespondingEthPeer.Responder emptyResponder = RespondingEthPeer.emptyResponder();
    peer.respondWhileOtherThreadsWork(emptyResponder, () -> !result.isDone());
    assertThat(result).isCompletedExceptionally();
    assertThatThrownBy(result::get).hasCauseInstanceOf(StalledDownloadException.class);
    // Finally, check that when we restart the download with state that is available it works
    final CompletableFuture<Void> retryResult = downloader.run(null, new FastSyncState(header));
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    peer.respondWhileOtherThreadsWork(responder, () -> !retryResult.isDone());
    assertThat(retryResult).isCompleted();
}

Example 4 with WorldStateKeyValueStorage

use of org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method canRecoverFromTimeouts.

@Test
public void canRecoverFromTimeouts() {
    final DeterministicEthScheduler.TimeoutPolicy timeoutPolicy = DeterministicEthScheduler.TimeoutPolicy.timeoutXTimes(2);
    final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(timeoutPolicy);
    final MockExecutorService serviceExecutor = ((DeterministicEthScheduler) ethProtocolManager.ethContext().getScheduler()).mockServiceExecutor();
    serviceExecutor.setAutoRun(false);
    // Setup "remote" state
    final WorldStateArchive remoteWorldStateArchive = createInMemoryWorldStateArchive();
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    final List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, 20);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Create some peers
    final List<RespondingEthPeer> peers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(5).collect(Collectors.toList());
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection);
    final FastSyncState fastSyncState = new FastSyncState(header);
    final CompletableFuture<Void> result = downloader.run(null, fastSyncState);
    serviceExecutor.runPendingFuturesInSeparateThreads(persistenceThread);
    // Respond to node data requests
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    respondUntilDone(peers, responder, result);
    // Check that all expected account data was downloaded
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertThat(result).isDone();
    assertAccountsMatch(localWorldState, accounts);
}

Example 5 with WorldStateKeyValueStorage

use of org.hyperledger.besu.ethereum.storage.keyvalue.WorldStateKeyValueStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method resumesFromNonEmptyQueue.

@Test
public void resumesFromNonEmptyQueue() {
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, 10);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Add some nodes to the taskCollection
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = spy(new InMemoryTasksPriorityQueues<>());
    List<Bytes32> queuedHashes = getFirstSetOfChildNodeRequests(remoteStorage, stateRoot);
    // Sanity check
    assertThat(queuedHashes.size()).isGreaterThan(0);
    for (Bytes32 bytes32 : queuedHashes) {
        taskCollection.add(new AccountTrieNodeDataRequest(Hash.wrap(bytes32), Optional.empty()));
    }
    // Sanity check
    for (final Bytes32 bytes32 : queuedHashes) {
        final Hash hash = Hash.wrap(bytes32);
        verify(taskCollection, times(1)).add(argThat((r) -> r.getHash().equals(hash)));
    }
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create a peer that can respond
    final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber());
    // Respond to node data requests
    final List<MessageData> sentMessages = new ArrayList<>();
    final RespondingEthPeer.Responder blockChainResponder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    final RespondingEthPeer.Responder responder = RespondingEthPeer.wrapResponderWithCollector(blockChainResponder, sentMessages);
    CompletableFuture<Void> result = downloader.run(null, new FastSyncState(header));
    peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone());
    assertThat(localStorage.isWorldStateAvailable(stateRoot, header.getHash())).isTrue();
    // Check that already enqueued trie nodes were requested
    final List<Bytes32> requestedHashes = sentMessages.stream().filter(m -> m.getCode() == EthPV63.GET_NODE_DATA).map(GetNodeDataMessage::readFrom).flatMap(m -> StreamSupport.stream(m.hashes().spliterator(), true)).collect(Collectors.toList());
    assertThat(requestedHashes.size()).isGreaterThan(0);
    assertThat(requestedHashes).containsAll(queuedHashes);
    // Check that already enqueued requests were not enqueued more than once
    for (Bytes32 bytes32 : queuedHashes) {
        final Hash hash = Hash.wrap(bytes32);
        verify(taskCollection, times(1)).add(argThat((r) -> r.getHash().equals(hash)));
    }
    // Check that all expected account data was downloaded
    assertThat(result).isDone();
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertAccountsMatch(localWorldState, accounts);
}