use of org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader in project besu by hyperledger.
the class FastDownloaderFactory method create.
public static Optional<FastSyncDownloader> create(final SynchronizerConfiguration syncConfig, final Path dataDirectory, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, final WorldStateStorage worldStateStorage, final SyncState syncState, final Clock clock) {
final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
final FastSyncStateStorage fastSyncStateStorage = new FastSyncStateStorage(fastSyncDataDirectory);
if (syncConfig.getSyncMode() != SyncMode.FAST) {
if (fastSyncStateStorage.isFastSyncInProgress()) {
throw new IllegalStateException("Unable to change the sync mode when fast sync is incomplete, please restart with fast sync mode");
} else {
return Optional.empty();
}
}
ensureDirectoryExists(fastSyncDataDirectory.toFile());
final FastSyncState fastSyncState = fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
if (fastSyncState.getPivotBlockHeader().isEmpty() && protocolContext.getBlockchain().getChainHeadBlockNumber() != BlockHeader.GENESIS_BLOCK_NUMBER) {
LOG.info("Fast sync was requested, but cannot be enabled because the local blockchain is not empty.");
return Optional.empty();
}
if (worldStateStorage instanceof BonsaiWorldStateKeyValueStorage) {
worldStateStorage.clear();
} else {
final Path queueDataDir = fastSyncDataDirectory.resolve("statequeue");
if (queueDataDir.toFile().exists()) {
LOG.warn("Fast sync is picking up after old fast sync version. Pruning the world state and starting from scratch.");
clearOldFastSyncWorldStateData(worldStateStorage, queueDataDir);
}
}
final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = createWorldStateDownloaderTaskCollection(metricsSystem, syncConfig.getWorldStateTaskCacheSize());
final WorldStateDownloader worldStateDownloader = new FastWorldStateDownloader(ethContext, worldStateStorage, taskCollection, syncConfig.getWorldStateHashCountPerRequest(), syncConfig.getWorldStateRequestParallelism(), syncConfig.getWorldStateMaxRequestsWithoutProgress(), syncConfig.getWorldStateMinMillisBeforeStalling(), clock, metricsSystem);
final FastSyncDownloader fastSyncDownloader = new FastSyncDownloader(new FastSyncActions(syncConfig, protocolSchedule, protocolContext, ethContext, syncState, metricsSystem), worldStateStorage, worldStateDownloader, fastSyncStateStorage, taskCollection, fastSyncDataDirectory, fastSyncState);
syncState.setWorldStateDownloadStatus(worldStateDownloader);
return Optional.of(fastSyncDownloader);
}
use of org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader in project besu by hyperledger.
the class CheckpointDownloaderFactory method createCheckpointDownloader.
public static Optional<FastSyncDownloader<?>> createCheckpointDownloader(final PivotBlockSelector pivotBlockSelector, final SynchronizerConfiguration syncConfig, final Path dataDirectory, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, final WorldStateStorage worldStateStorage, final SyncState syncState, final Clock clock) {
final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
final FastSyncStateStorage fastSyncStateStorage = new FastSyncStateStorage(fastSyncDataDirectory);
if (SyncMode.isFullSync(syncConfig.getSyncMode())) {
if (fastSyncStateStorage.isFastSyncInProgress()) {
throw new IllegalStateException("Unable to change the sync mode when snap sync is incomplete, please restart with checkpoint sync mode");
} else {
return Optional.empty();
}
}
ensureDirectoryExists(fastSyncDataDirectory.toFile());
final FastSyncState fastSyncState = fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
if (fastSyncState.getPivotBlockHeader().isEmpty() && protocolContext.getBlockchain().getChainHeadBlockNumber() != BlockHeader.GENESIS_BLOCK_NUMBER) {
LOG.info("Checkpoint sync was requested, but cannot be enabled because the local blockchain is not empty.");
return Optional.empty();
}
final FastSyncActions fastSyncActions;
if (syncState.getCheckpoint().isEmpty()) {
LOG.warn("Unable to find a valid checkpoint configuration. The genesis will be used");
fastSyncActions = new FastSyncActions(syncConfig, worldStateStorage, protocolSchedule, protocolContext, ethContext, syncState, pivotBlockSelector, metricsSystem);
} else {
LOG.info("Checkpoint sync start with block {} and hash {}", syncState.getCheckpoint().get().blockNumber(), syncState.getCheckpoint().get().blockHash());
fastSyncActions = new CheckpointSyncActions(syncConfig, worldStateStorage, protocolSchedule, protocolContext, ethContext, syncState, pivotBlockSelector, metricsSystem);
}
final SnapSyncState snapSyncState = new SnapSyncState(fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule)));
worldStateStorage.clear();
final InMemoryTasksPriorityQueues<SnapDataRequest> snapTaskCollection = createSnapWorldStateDownloaderTaskCollection();
final WorldStateDownloader snapWorldStateDownloader = new SnapWorldStateDownloader(ethContext, worldStateStorage, snapTaskCollection, syncConfig.getSnapSyncConfiguration(), syncConfig.getWorldStateRequestParallelism(), syncConfig.getWorldStateMaxRequestsWithoutProgress(), syncConfig.getWorldStateMinMillisBeforeStalling(), clock, metricsSystem);
final FastSyncDownloader<SnapDataRequest> fastSyncDownloader = new SnapSyncDownloader(fastSyncActions, worldStateStorage, snapWorldStateDownloader, fastSyncStateStorage, snapTaskCollection, fastSyncDataDirectory, snapSyncState);
syncState.setWorldStateDownloadStatus(snapWorldStateDownloader);
return Optional.of(fastSyncDownloader);
}
use of org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader in project besu by hyperledger.
the class FastWorldStateDownloaderTest method testCancellation.
@SuppressWarnings("unchecked")
private void testCancellation(final boolean shouldCancelFuture) {
final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create();
// Prevent the persistence service from running
final MockExecutorService serviceExecutor = ((DeterministicEthScheduler) ethProtocolManager.ethContext().getScheduler()).mockServiceExecutor();
serviceExecutor.setAutoRun(false);
// Setup "remote" state
final WorldStateArchive remoteWorldStateArchive = createInMemoryWorldStateArchive();
final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
// Generate accounts and save corresponding state root
dataGen.createRandomContractAccountsWithNonEmptyStorage(remoteWorldState, 20);
final Hash stateRoot = remoteWorldState.rootHash();
final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
// Create some peers
final List<RespondingEthPeer> peers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(5).collect(Collectors.toList());
final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
final WorldStateDownloader downloader = createDownloader(ethProtocolManager.ethContext(), localStorage, taskCollection);
final FastSyncState fastSyncState = new FastSyncState(header);
final CompletableFuture<Void> result = downloader.run(null, fastSyncState);
// Send a few responses
final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
for (int i = 0; i < 3; i++) {
for (final RespondingEthPeer peer : peers) {
peer.respond(responder);
}
giveOtherThreadsAGo();
}
// Sanity check
assertThat(result.isDone()).isFalse();
// Reset taskCollection so we can track interactions after the cancellation
reset(taskCollection);
if (shouldCancelFuture) {
result.cancel(true);
} else {
downloader.cancel();
assertThat(result).isCancelled();
}
// Send some more responses after cancelling
for (int i = 0; i < 3; i++) {
for (final RespondingEthPeer peer : peers) {
peer.respond(responder);
}
giveOtherThreadsAGo();
}
// Now allow the persistence service to run which should exit immediately
serviceExecutor.runPendingFutures();
verify(taskCollection, times(1)).clear();
verify(taskCollection, never()).remove();
verify(taskCollection, never()).add(any(NodeDataRequest.class));
// Target world state should not be available
assertThat(localStorage.isWorldStateAvailable(header.getStateRoot(), header.getHash())).isFalse();
}
use of org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader in project besu by hyperledger.
the class FastWorldStateDownloaderTest method downloadAvailableWorldStateFromPeers.
private void downloadAvailableWorldStateFromPeers(final int peerCount, final int accountCount, final int hashesPerRequest, final int maxOutstandingRequests, final NetworkResponder networkResponder) {
final int trailingPeerCount = 5;
// Setup "remote" state
final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
// Generate accounts and save corresponding state root
final List<Account> accounts = dataGen.createRandomAccounts(remoteWorldState, accountCount);
final Hash stateRoot = remoteWorldState.rootHash();
// Sanity check
assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
// Generate more data that should not be downloaded
final List<Account> otherAccounts = dataGen.createRandomAccounts(remoteWorldState, 5);
final Hash otherStateRoot = remoteWorldState.rootHash();
final BlockHeader otherHeader = dataGen.block(BlockOptions.create().setStateRoot(otherStateRoot).setBlockNumber(11)).getHeader();
// Sanity check
assertThat(otherStateRoot).isNotEqualTo(stateRoot);
final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
final WorldStateArchive localWorldStateArchive = new DefaultWorldStateArchive(localStorage, createPreimageStorage());
final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateHashCountPerRequest(hashesPerRequest).worldStateRequestParallelism(maxOutstandingRequests).build();
final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
// Create some peers that can respond
final List<RespondingEthPeer> usefulPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber())).limit(peerCount).collect(Collectors.toList());
// And some irrelevant peers
final List<RespondingEthPeer> trailingPeers = Stream.generate(() -> EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber() - 1L)).limit(trailingPeerCount).collect(Collectors.toList());
// Start downloader
final CompletableFuture<?> result = downloader.run(null, new FastSyncState(header));
// A second run should return an error without impacting the first result
final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
assertThat(secondResult).isCompletedExceptionally();
assertThat(result).isNotCompletedExceptionally();
// Respond to node data requests
// Send one round of full responses, so that we can get multiple requests queued up
final RespondingEthPeer.Responder fullResponder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
for (final RespondingEthPeer peer : usefulPeers) {
peer.respond(fullResponder);
}
// Respond to remaining queued requests in custom way
networkResponder.respond(usefulPeers, remoteWorldStateArchive, result);
// Check that trailing peers were not queried for data
for (final RespondingEthPeer trailingPeer : trailingPeers) {
assertThat(trailingPeer.hasOutstandingRequests()).isFalse();
}
// Check that all expected account data was downloaded
final WorldState localWorldState = localWorldStateArchive.get(stateRoot, null).get();
assertThat(result).isDone();
assertAccountsMatch(localWorldState, accounts);
// We shouldn't have any extra data locally
assertThat(localStorage.contains(otherHeader.getStateRoot())).isFalse();
for (final Account otherAccount : otherAccounts) {
assertThat(localWorldState.get(otherAccount.getAddress())).isNull();
}
}
use of org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader in project besu by hyperledger.
the class FastWorldStateDownloaderTest method stalledDownloader.
@Test
public void stalledDownloader() {
final EthProtocolManager ethProtocolManager = EthProtocolManagerTestUtil.create(new EthScheduler(1, 1, 1, 1, new NoOpMetricsSystem()));
// Setup "remote" state
final WorldStateStorage remoteStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
final WorldStateArchive remoteWorldStateArchive = new DefaultWorldStateArchive(remoteStorage, createPreimageStorage());
final MutableWorldState remoteWorldState = remoteWorldStateArchive.getMutable();
// Generate accounts and save corresponding state root
dataGen.createRandomAccounts(remoteWorldState, 10);
final Hash stateRoot = remoteWorldState.rootHash();
// Sanity check
assertThat(stateRoot).isNotEqualTo(EMPTY_TRIE_ROOT);
final BlockHeader header = dataGen.block(BlockOptions.create().setStateRoot(stateRoot).setBlockNumber(10)).getHeader();
final InMemoryTasksPriorityQueues<NodeDataRequest> taskCollection = new InMemoryTasksPriorityQueues<>();
final WorldStateStorage localStorage = new WorldStateKeyValueStorage(new InMemoryKeyValueStorage());
final SynchronizerConfiguration syncConfig = SynchronizerConfiguration.builder().worldStateMaxRequestsWithoutProgress(10).build();
final WorldStateDownloader downloader = createDownloader(syncConfig, ethProtocolManager.ethContext(), localStorage, taskCollection);
// Create a peer that can respond
final RespondingEthPeer peer = EthProtocolManagerTestUtil.createPeer(ethProtocolManager, header.getNumber());
// Start downloader (with a state root that's not available anywhere
final CompletableFuture<Void> result = downloader.run(null, new FastSyncState(new BlockHeaderTestFixture().stateRoot(Hash.hash(Bytes.of(1, 2, 3, 4))).buildHeader()));
// A second run should return an error without impacting the first result
final CompletableFuture<?> secondResult = downloader.run(null, new FastSyncState(header));
assertThat(secondResult).isCompletedExceptionally();
assertThat(result).isNotCompletedExceptionally();
final RespondingEthPeer.Responder emptyResponder = RespondingEthPeer.emptyResponder();
peer.respondWhileOtherThreadsWork(emptyResponder, () -> !result.isDone());
assertThat(result).isCompletedExceptionally();
assertThatThrownBy(result::get).hasCauseInstanceOf(StalledDownloadException.class);
// Finally, check that when we restart the download with state that is available it works
final CompletableFuture<Void> retryResult = downloader.run(null, new FastSyncState(header));
final RespondingEthPeer.Responder responder = RespondingEthPeer.blockchainResponder(mock(Blockchain.class), remoteWorldStateArchive);
peer.respondWhileOtherThreadsWork(responder, () -> !retryResult.isDone());
assertThat(retryResult).isCompleted();
}
Aggregations