Examples with SynchronizerConfiguration org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration used on opensource projects

Example 1 with SynchronizerConfiguration

use of org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration in project besu by hyperledger.

the class FastWorldStateDownloaderTest method downloadAvailableWorldStateFromPeers.

private void downloadAvailableWorldStateFromPeers(final int peerCount, final int accountCount, final int hashesPerRequest, final int maxOutstandingRequests, final NetworkResponder networkResponder) {
    final int trailingPeerCount = 5;
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    final List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, accountCount);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Generate more data that should not be downloaded
    final List<Account> otherAccounts = dataGen.createRandomAccounts(remoteWorldState, 5);
    final Hash otherStateRoot = remoteWorldState.rootHash();
    final BlockHeader otherHeader = dataGen.block(BlockOptions.create().setStateRoot(otherStateRoot).setBlockNumber(11)).getHeader();
    // Sanity check
    assertThat(otherStateRoot).isNotEqualTo(stateRoot);
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateHashCountPerRequest(hashesPerRequest).worldStateRequestParallelism(maxOutstandingRequests).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create some peers that can respond
    final List<RespondingEthPeer> usefulPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(peerCount).collect(Collectors.toList());
    // And some irrelevant peers
    final List<RespondingEthPeer> trailingPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber() - 1L)).limit(trailingPeerCount).collect(Collectors.toList());
    // Start downloader
    final CompletableFuture<?> result = downloader.run(null, new FastSyncState(header));
    // A second run should return an error without impacting the first result
    final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
    assertThat(secondResult).isCompletedExceptionally();
    assertThat(result).isNotCompletedExceptionally();
    // Respond to node data requests
    // Send one round of full responses, so that we can get multiple requests queued up
    final RespondingEthPeer.Responder fullResponder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    for (final RespondingEthPeer peer : usefulPeers) {
        peer.respond(fullResponder);
    }
    // Respond to remaining queued requests in custom way
    networkResponder.respond(usefulPeers, remoteWorldStateArchive, result);
    // Check that trailing peers were not queried for data
    for (final RespondingEthPeer trailingPeer : trailingPeers) {
        assertThat(trailingPeer.hasOutstandingRequests()).isFalse();
    }
    // Check that all expected account data was downloaded
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertThat(result).isDone();
    assertAccountsMatch(localWorldState, accounts);
    // We shouldn't have any extra data locally
    assertThat(localStorage.contains(otherHeader.getStateRoot())).isFalse();
    for (final Account otherAccount : otherAccounts) {
        assertThat(localWorldState.get(otherAccount.getAddress())).isNull();
    }
}

Example 2 with SynchronizerConfiguration

use of org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration in project besu by hyperledger.

the class FastWorldStateDownloaderTest method stalledDownloader.

@Test
public void stalledDownloader() {
    final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(new EthScheduler(1, 1, 1, 1, new NoOpMetricsSystem()));
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    dataGen.createRandomAccounts(remoteWorldState, 10);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create a peer that can respond
    final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber());
    // Start downloader (with a state root that's not available anywhere
    final CompletableFuture<Void> result = downloader.run(null, new FastSyncState(new BlockHeaderTestFixture().stateRoot(Hash.hash(Bytes.of(1, 2, 3, 4))).buildHeader()));
    // A second run should return an error without impacting the first result
    final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
    assertThat(secondResult).isCompletedExceptionally();
    assertThat(result).isNotCompletedExceptionally();
    final RespondingEthPeer.Responder emptyResponder = RespondingEthPeer.emptyResponder();
    peer.respondWhileOtherThreadsWork(emptyResponder, () -> !result.isDone());
    assertThat(result).isCompletedExceptionally();
    assertThatThrownBy(result::get).hasCauseInstanceOf(StalledDownloadException.class);
    // Finally, check that when we restart the download with state that is available it works
    final CompletableFuture<Void> retryResult = downloader.run(null, new FastSyncState(header));
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    peer.respondWhileOtherThreadsWork(responder, () -> !retryResult.isDone());
    assertThat(retryResult).isCompleted();
}

Example 3 with SynchronizerConfiguration

use of org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration in project besu by hyperledger.

the class FastWorldStateDownloaderTest method resumesFromNonEmptyQueue.

@Test
public void resumesFromNonEmptyQueue() {
    // Setup "remote" state
    final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
    final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
    // Generate accounts and save corresponding state root
    List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, 10);
    final Hash stateRoot = remoteWorldState.rootHash();
    // Sanity check
    assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
    final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
    // Add some nodes to the taskCollection
    final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = spy(new InMemoryTasksPriorityQueues<>());
    List<Bytes32> queuedHashes = getFirstSetOfChildNodeRequests(remoteStorage, stateRoot);
    // Sanity check
    assertThat(queuedHashes.size()).isGreaterThan(0);
    for (Bytes32 bytes32 : queuedHashes) {
        taskCollection.add(new AccountTrieNodeDataRequest(Hash.wrap(bytes32), Optional.empty()));
    }
    // Sanity check
    for (final Bytes32 bytes32 : queuedHashes) {
        final Hash hash = Hash.wrap(bytes32);
        verify(taskCollection, times(1)).add(argThat((r) -> r.getHash().equals(hash)));
    }
    final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build();
    final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
    // Create a peer that can respond
    final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber());
    // Respond to node data requests
    final List<MessageData> sentMessages = new ArrayList<>();
    final RespondingEthPeer.Responder blockChainResponder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
    final RespondingEthPeer.Responder responder = RespondingEthPeer.wrapResponderWithCollector(blockChainResponder, sentMessages);
    CompletableFuture<Void> result = downloader.run(null, new FastSyncState(header));
    peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone());
    assertThat(localStorage.isWorldStateAvailable(stateRoot, header.getHash())).isTrue();
    // Check that already enqueued trie nodes were requested
    final List<Bytes32> requestedHashes = sentMessages.stream().filter(m -> m.getCode() == EthPV63.GET_NODE_DATA).map(GetNodeDataMessage::readFrom).flatMap(m -> StreamSupport.stream(m.hashes().spliterator(), true)).collect(Collectors.toList());
    assertThat(requestedHashes.size()).isGreaterThan(0);
    assertThat(requestedHashes).containsAll(queuedHashes);
    // Check that already enqueued requests were not enqueued more than once
    for (Bytes32 bytes32 : queuedHashes) {
        final Hash hash = Hash.wrap(bytes32);
        verify(taskCollection, times(1)).add(argThat((r) -> r.getHash().equals(hash)));
    }
    // Check that all expected account data was downloaded
    assertThat(result).isDone();
    final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
    final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
    assertAccountsMatch(localWorldState, accounts);
}

Example 4 with SynchronizerConfiguration

use of org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration in project besu by hyperledger.

the class FastSyncChainDownloaderTest method shouldSyncToPivotBlockInSingleSegment.

@Test
public void shouldSyncToPivotBlockInSingleSegment() {
    otherBlockchainSetup.importFirstBlocks(30);
    final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, otherBlockchain);
    final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(otherBlockchain);
    final long pivotBlockNumber = 5;
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().build();
    final ChainDownloader downloader = downloader(syncConfig, pivotBlockNumber);
    final CompletableFuture<Void> result = downloader.start();
    peer.respondWhileOtherThreadsWork(responder, () -> !result.isDone());
    assertThat(result).isCompleted();
    assertThat(localBlockchain.getChainHeadBlockNumber()).isEqualTo(pivotBlockNumber);
    assertThat(localBlockchain.getChainHeadHeader()).isEqualTo(otherBlockchain.getBlockHeader(pivotBlockNumber).get());
}

Example 5 with SynchronizerConfiguration

use of org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration in project besu by hyperledger.

the class FastSyncChainDownloaderTest method recoversFromSyncTargetDisconnect.

@Test
public void recoversFromSyncTargetDisconnect() {
    final BlockchainSetupUtil shorterChainUtil = BlockchainSetupUtil.forTesting(storageFormat);
    final MutableBlockchain shorterChain = shorterChainUtil.getBlockchain();
    otherBlockchainSetup.importFirstBlocks(30);
    shorterChainUtil.importFirstBlocks(28);
    final RespondingEthPeer bestPeer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, otherBlockchain);
    final RespondingEthPeer secondBestPeer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, shorterChain);
    final RespondingEthPeer.Responder shorterResponder = RespondingEthPeer.blockchainResponder(shorterChain);
    // Doesn't respond to requests for checkpoints unless it's starting from genesis
    // So the import can only make it as far as block 15 (3 checkpoints 5 blocks apart)
    final RespondingEthPeer.Responder shorterLimitedRangeResponder = RespondingEthPeer.targetedResponder((cap, msg) -> {
        if (msg.getCode() == EthPV62.GET_BLOCK_HEADERS) {
            final GetBlockHeadersMessage request = GetBlockHeadersMessage.readFrom(msg);
            return request.skip() == 0 || (request.hash().equals(localBlockchain.getBlockHashByNumber(0)));
        } else {
            return true;
        }
    }, (cap, msg) -> shorterResponder.respond(cap, msg).get());
    final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().downloaderChainSegmentSize(5).downloaderHeadersRequestSize(3).downloaderParallelism(1).build();
    final long pivotBlockNumber = 25;
    final ChainDownloader downloader = downloader(syncConfig, pivotBlockNumber);
    final CompletableFuture<Void> result = downloader.start();
    while (localBlockchain.getChainHeadBlockNumber() < 15) {
        bestPeer.respond(shorterLimitedRangeResponder);
        secondBestPeer.respond(shorterLimitedRangeResponder);
        LockSupport.parkNanos(200);
    }
    assertThat(localBlockchain.getChainHeadBlockNumber()).isEqualTo(15);
    assertThat(result).isNotCompleted();
    ethProtocolManager.handleDisconnect(bestPeer.getPeerConnection(), TOO_MANY_PEERS, true);
    secondBestPeer.respondWhileOtherThreadsWork(shorterResponder, () -> !result.isDone());
    assertThat(result).isCompleted();
    assertThat(localBlockchain.getChainHeadBlockNumber()).isEqualTo(pivotBlockNumber);
    assertThat(localBlockchain.getChainHeadHeader()).isEqualTo(otherBlockchain.getBlockHeader(pivotBlockNumber).get());
}