Examples with WorldStateStorage org.hyperledger.besu.ethereum.worldstate.WorldStateStorage used on opensource projects

Example 1 with WorldStateStorage

use of org.hyperledger.besu.ethereum.worldstate.WorldStateStorage in project besu by hyperledger.

the class TrieGenerator method generateTrie.

public static MerklePatriciaTrie<Bytes32, Bytes> generateTrie(final WorldStateStorage worldStateStorage, final int nbAccounts) {
    final List<Hash> accountHash = new ArrayList<>();
    final MerklePatriciaTrie<Bytes32, Bytes> accountStateTrie = emptyAccountStateTrie(worldStateStorage);
    // Add some storage values
    for (int i = 0; i < nbAccounts; i++) {
        final WorldStateStorage.Updater updater = worldStateStorage.updater();
        accountHash.add(Hash.wrap(Bytes32.leftPad(Bytes.of(i + 1))));
        final MerklePatriciaTrie<Bytes32, Bytes> storageTrie = emptyStorageTrie(worldStateStorage, accountHash.get(i));
        writeStorageValue(storageTrie, UInt256.ONE, UInt256.valueOf(2L));
        writeStorageValue(storageTrie, UInt256.valueOf(2L), UInt256.valueOf(4L));
        writeStorageValue(storageTrie, UInt256.valueOf(3L), UInt256.valueOf(6L));
        int accountIndex = i;
        storageTrie.commit((location, hash, value) -> updater.putAccountStorageTrieNode(accountHash.get(accountIndex), location, hash, value));
        final Bytes code = Bytes32.leftPad(Bytes.of(i + 10));
        final Hash codeHash = Hash.hash(code);
        final StateTrieAccountValue accountValue = new StateTrieAccountValue(1L, Wei.of(2L), Hash.wrap(storageTrie.getRootHash()), codeHash);
        accountStateTrie.put(accountHash.get(i), RLP.encode(accountValue::writeTo));
        accountStateTrie.commit(updater::putAccountStateTrieNode);
        updater.putCode(codeHash, code);
        // Persist updates
        updater.commit();
    }
    return accountStateTrie;
}

Example 2 with WorldStateStorage

use of org.hyperledger.besu.ethereum.worldstate.WorldStateStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method testCancellation.

@SuppressWarnings("unchecked")
private void testCancellation(final boolean shouldCancelFuture) {
    final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create();
    // Prevent the persistence service from running
    final MockExecutorService serviceExecutor = ((DeterministicEthScheduler) ethProtocolManager.ethContext().getScheduler()).mockServiceExecutor();
    serviceExecutor.setAutoRun(false);
    // Setup "remote" state
    final WorldStateArchive remoteWorldStateArchive = createInMemoryWorldStateArchive();
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    dataGen.createRandomContractAccountsWithNonEmptyStorage(remoteWorldState, 20);
    final Hash stateRoot = remoteWorldState.rootHash();
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Create some peers
    final List<RespondingEthPeer> peers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(5).collect(Collectors.toList());
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection);
    final FastSyncState fastSyncState = new FastSyncState(header);
    final CompletableFuture<Void> result = downloader.run(null, fastSyncState);
    // Send a few responses
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    for (int i = 0; i < 3; i++) {
        for (final RespondingEthPeer peer : peers) {
            peer.respond(responder);
        }
        giveOtherThreadsAGo();
    }
    // Sanity check
    assertThat(result.isDone()).isFalse();
    // Reset taskCollection so we can track interactions after the cancellation
    reset(taskCollection);
    if (shouldCancelFuture) {
        result.cancel(true);
    } else {
        downloader.cancel();
        assertThat(result).isCancelled();
    }
    // Send some more responses after cancelling
    for (int i = 0; i < 3; i++) {
        for (final RespondingEthPeer peer : peers) {
            peer.respond(responder);
        }
        giveOtherThreadsAGo();
    }
    // Now allow the persistence service to run which should exit immediately
    serviceExecutor.runPendingFutures();
    verify(taskCollection, times(1)).clear();
    verify(taskCollection, never()).remove();
    verify(taskCollection, never()).add(any(NodeDataRequest.class));
    // Target world state should not be available
    assertThat(localStorage.isWorldStateAvailable(header.getStateRoot(), header.getHash())).isFalse();
}

Example 3 with WorldStateStorage

use of org.hyperledger.besu.ethereum.worldstate.WorldStateStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method downloadAvailableWorldStateFromPeers.

private void downloadAvailableWorldStateFromPeers(final int peerCount, final int accountCount, final int hashesPerRequest, final int maxOutstandingRequests, final NetworkResponder networkResponder) {
    final int trailingPeerCount = 5;
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    final List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, accountCount);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Generate more data that should not be downloaded
    final List<Account> otherAccounts = dataGen.createRandomAccounts(remoteWorldState, 5);
    final Hash otherStateRoot = remoteWorldState.rootHash();
    final BlockHeader otherHeader = dataGen.block(BlockOptions.create().setStateRoot(otherStateRoot).setBlockNumber(11)).getHeader();
    // Sanity check
    assertThat(otherStateRoot).isNotEqualTo(stateRoot);
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateHashCountPerRequest(hashesPerRequest).worldStateRequestParallelism(maxOutstandingRequests).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create some peers that can respond
    final List<RespondingEthPeer> usefulPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(peerCount).collect(Collectors.toList());
    // And some irrelevant peers
    final List<RespondingEthPeer> trailingPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber() - 1L)).limit(trailingPeerCount).collect(Collectors.toList());
    // Start downloader
    final CompletableFuture<?> result = downloader.run(null, new FastSyncState(header));
    // A second run should return an error without impacting the first result
    final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
    assertThat(secondResult).isCompletedExceptionally();
    assertThat(result).isNotCompletedExceptionally();
    // Respond to node data requests
    // Send one round of full responses, so that we can get multiple requests queued up
    final RespondingEthPeer.Responder fullResponder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    for (final RespondingEthPeer peer : usefulPeers) {
        peer.respond(fullResponder);
    }
    // Respond to remaining queued requests in custom way
    networkResponder.respond(usefulPeers, remoteWorldStateArchive, result);
    // Check that trailing peers were not queried for data
    for (final RespondingEthPeer trailingPeer : trailingPeers) {
        assertThat(trailingPeer.hasOutstandingRequests()).isFalse();
    }
    // Check that all expected account data was downloaded
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertThat(result).isDone();
    assertAccountsMatch(localWorldState, accounts);
    // We shouldn't have any extra data locally
    assertThat(localStorage.contains(otherHeader.getStateRoot())).isFalse();
    for (final Account otherAccount : otherAccounts) {
        assertThat(localWorldState.get(otherAccount.getAddress())).isNull();
    }
}

Example 4 with WorldStateStorage

use of org.hyperledger.besu.ethereum.worldstate.WorldStateStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method stalledDownloader.

@Test
public void stalledDownloader() {
    final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(new EthScheduler(1, 1, 1, 1, new NoOpMetricsSystem()));
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    dataGen.createRandomAccounts(remoteWorldState, 10);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create a peer that can respond
    final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber());
    // Start downloader (with a state root that's not available anywhere
    final CompletableFuture<Void> result = downloader.run(null, new FastSyncState(new BlockHeaderTestFixture().stateRoot(Hash.hash(Bytes.of(1, 2, 3, 4))).buildHeader()));
    // A second run should return an error without impacting the first result
    final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
    assertThat(secondResult).isCompletedExceptionally();
    assertThat(result).isNotCompletedExceptionally();
    final RespondingEthPeer.Responder emptyResponder = RespondingEthPeer.emptyResponder();
    peer.respondWhileOtherThreadsWork(emptyResponder, () -> !result.isDone());
    assertThat(result).isCompletedExceptionally();
    assertThatThrownBy(result::get).hasCauseInstanceOf(StalledDownloadException.class);
    // Finally, check that when we restart the download with state that is available it works
    final CompletableFuture<Void> retryResult = downloader.run(null, new FastSyncState(header));
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    peer.respondWhileOtherThreadsWork(responder, () -> !retryResult.isDone());
    assertThat(retryResult).isCompleted();
}

Example 5 with WorldStateStorage

use of org.hyperledger.besu.ethereum.worldstate.WorldStateStorage in project besu by hyperledger.

the class FastWorldStateDownloaderTest method canRecoverFromTimeouts.

@Test
public void canRecoverFromTimeouts() {
    final DeterministicEthScheduler.TimeoutPolicy timeoutPolicy = DeterministicEthScheduler.TimeoutPolicy.timeoutXTimes(2);
    final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(timeoutPolicy);
    final MockExecutorService serviceExecutor = ((DeterministicEthScheduler) ethProtocolManager.ethContext().getScheduler()).mockServiceExecutor();
    serviceExecutor.setAutoRun(false);
    // Setup "remote" state
    final WorldStateArchive remoteWorldStateArchive = createInMemoryWorldStateArchive();
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    final List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, 20);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Create some peers
    final List<RespondingEthPeer> peers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(5).collect(Collectors.toList());
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection);
    final FastSyncState fastSyncState = new FastSyncState(header);
    final CompletableFuture<Void> result = downloader.run(null, fastSyncState);
    serviceExecutor.runPendingFuturesInSeparateThreads(persistenceThread);
    // Respond to node data requests
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    respondUntilDone(peers, responder, result);
    // Check that all expected account data was downloaded
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertThat(result).isDone();
    assertAccountsMatch(localWorldState, accounts);
}