Search in sources :

Example 1 with SnapDataRequest

use of org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest in project besu by hyperledger.

the class CheckpointDownloaderFactory method createCheckpointDownloader.

public static Optional<FastSyncDownloader<?>> createCheckpointDownloader(final PivotBlockSelector pivotBlockSelector, final SynchronizerConfiguration syncConfig, final Path dataDirectory, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, final WorldStateStorage worldStateStorage, final SyncState syncState, final Clock clock) {
    final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
    final FastSyncStateStorage fastSyncStateStorage = new FastSyncStateStorage(fastSyncDataDirectory);
    if (SyncMode.isFullSync(syncConfig.getSyncMode())) {
        if (fastSyncStateStorage.isFastSyncInProgress()) {
            throw new IllegalStateException("Unable to change the sync mode when snap sync is incomplete, please restart with checkpoint sync mode");
        } else {
            return Optional.empty();
        }
    }
    ensureDirectoryExists(fastSyncDataDirectory.toFile());
    final FastSyncState fastSyncState = fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
    if (fastSyncState.getPivotBlockHeader().isEmpty() && protocolContext.getBlockchain().getChainHeadBlockNumber() != BlockHeader.GENESIS_BLOCK_NUMBER) {
        LOG.info("Checkpoint sync was requested, but cannot be enabled because the local blockchain is not empty.");
        return Optional.empty();
    }
    final FastSyncActions fastSyncActions;
    if (syncState.getCheckpoint().isEmpty()) {
        LOG.warn("Unable to find a valid checkpoint configuration. The genesis will be used");
        fastSyncActions = new FastSyncActions(syncConfig, worldStateStorage, protocolSchedule, protocolContext, ethContext, syncState, pivotBlockSelector, metricsSystem);
    } else {
        LOG.info("Checkpoint sync start with block {} and hash {}", syncState.getCheckpoint().get().blockNumber(), syncState.getCheckpoint().get().blockHash());
        fastSyncActions = new CheckpointSyncActions(syncConfig, worldStateStorage, protocolSchedule, protocolContext, ethContext, syncState, pivotBlockSelector, metricsSystem);
    }
    final SnapSyncState snapSyncState = new SnapSyncState(fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule)));
    worldStateStorage.clear();
    final InMemoryTasksPriorityQueues<SnapDataRequest> snapTaskCollection = createSnapWorldStateDownloaderTaskCollection();
    final WorldStateDownloader snapWorldStateDownloader = new SnapWorldStateDownloader(ethContext, worldStateStorage, snapTaskCollection, syncConfig.getSnapSyncConfiguration(), syncConfig.getWorldStateRequestParallelism(), syncConfig.getWorldStateMaxRequestsWithoutProgress(), syncConfig.getWorldStateMinMillisBeforeStalling(), clock, metricsSystem);
    final FastSyncDownloader<SnapDataRequest> fastSyncDownloader = new SnapSyncDownloader(fastSyncActions, worldStateStorage, snapWorldStateDownloader, fastSyncStateStorage, snapTaskCollection, fastSyncDataDirectory, snapSyncState);
    syncState.setWorldStateDownloadStatus(snapWorldStateDownloader);
    return Optional.of(fastSyncDownloader);
}
Also used : Path(java.nio.file.Path) SnapDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest) SnapWorldStateDownloader(org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldStateDownloader) WorldStateDownloader(org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader) FastSyncState(org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState) SnapSyncState(org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncState) FastSyncActions(org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncActions) FastSyncStateStorage(org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncStateStorage) SnapSyncDownloader(org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapSyncDownloader) SnapWorldStateDownloader(org.hyperledger.besu.ethereum.eth.sync.snapsync.SnapWorldStateDownloader)

Example 2 with SnapDataRequest

use of org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest in project besu by hyperledger.

the class PersistDataStepTest method assertDataPersisted.

private void assertDataPersisted(final List<Task<SnapDataRequest>> tasks) {
    tasks.forEach(task -> {
        if (task.getData() instanceof AccountRangeDataRequest) {
            final AccountRangeDataRequest data = (AccountRangeDataRequest) task.getData();
            StoredMerklePatriciaTrie<Bytes, Bytes> trie = new StoredMerklePatriciaTrie<>(worldStateStorage::getAccountStateTrieNode, data.getRootHash(), b -> b, b -> b);
            data.getAccounts().forEach((key, value) -> assertThat(trie.get(key)).isPresent());
        } else if (task.getData() instanceof StorageRangeDataRequest) {
            final StorageRangeDataRequest data = (StorageRangeDataRequest) task.getData();
            final StoredMerklePatriciaTrie<Bytes, Bytes> trie = new StoredMerklePatriciaTrie<>((location, hash) -> worldStateStorage.getAccountStorageTrieNode(Hash.wrap(data.getAccountHash()), location, hash), data.getStorageRoot(), b -> b, b -> b);
            data.getSlots().forEach((key, value) -> assertThat(trie.get(key)).isPresent());
        } else if (task.getData() instanceof BytecodeRequest) {
            final BytecodeRequest data = (BytecodeRequest) task.getData();
            assertThat(worldStateStorage.getCode(data.getCodeHash(), Hash.wrap(data.getAccountHash()))).isPresent();
        } else {
            fail("not expected message");
        }
    });
}
Also used : StoredMerklePatriciaTrie(org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie) InMemoryKeyValueStorageProvider(org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Test(org.junit.Test) Mockito.when(org.mockito.Mockito.when) Bytes(org.apache.tuweni.bytes.Bytes) BytecodeRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest) SnapDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest) List(java.util.List) Assertions.fail(org.assertj.core.api.Assertions.fail) DataStorageFormat(org.hyperledger.besu.ethereum.worldstate.DataStorageFormat) AccountRangeDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataRequest) StorageRangeDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest) WorldStateStorage(org.hyperledger.besu.ethereum.worldstate.WorldStateStorage) Task(org.hyperledger.besu.services.tasks.Task) Mockito.mock(org.mockito.Mockito.mock) Hash(org.hyperledger.besu.datatypes.Hash) Before(org.junit.Before) Bytes(org.apache.tuweni.bytes.Bytes) StoredMerklePatriciaTrie(org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie) BytecodeRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest) AccountRangeDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataRequest) StorageRangeDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest)

Example 3 with SnapDataRequest

use of org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest in project besu by hyperledger.

the class RequestDataStep method requestCode.

public CompletableFuture<List<Task<SnapDataRequest>>> requestCode(final List<Task<SnapDataRequest>> requestTasks) {
    final List<Bytes32> codeHashes = requestTasks.stream().map(Task::getData).map(BytecodeRequest.class::cast).map(BytecodeRequest::getCodeHash).distinct().collect(Collectors.toList());
    final BlockHeader blockHeader = fastSyncState.getPivotBlockHeader().get();
    final EthTask<Map<Bytes32, Bytes>> getByteCodeTask = RetryingGetBytecodeFromPeerTask.forByteCode(ethContext, codeHashes, blockHeader, metricsSystem);
    downloadState.addOutstandingTask(getByteCodeTask);
    return getByteCodeTask.run().handle((response, error) -> {
        if (response != null) {
            downloadState.removeOutstandingTask(getByteCodeTask);
            for (Task<SnapDataRequest> requestTask : requestTasks) {
                final BytecodeRequest request = (BytecodeRequest) requestTask.getData();
                request.setRootHash(blockHeader.getStateRoot());
                if (response.containsKey(request.getCodeHash())) {
                    request.setCode(response.get(request.getCodeHash()));
                }
            }
        }
        return requestTasks;
    });
}
Also used : RetryingGetBytecodeFromPeerTask(org.hyperledger.besu.ethereum.eth.manager.snap.RetryingGetBytecodeFromPeerTask) EthTask(org.hyperledger.besu.ethereum.eth.manager.task.EthTask) RetryingGetTrieNodeFromPeerTask(org.hyperledger.besu.ethereum.eth.manager.snap.RetryingGetTrieNodeFromPeerTask) RetryingGetAccountRangeFromPeerTask(org.hyperledger.besu.ethereum.eth.manager.snap.RetryingGetAccountRangeFromPeerTask) Task(org.hyperledger.besu.services.tasks.Task) RetryingGetStorageRangeFromPeerTask(org.hyperledger.besu.ethereum.eth.manager.snap.RetryingGetStorageRangeFromPeerTask) SnapDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest) BytecodeRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest) BlockHeader(org.hyperledger.besu.ethereum.core.BlockHeader) Bytes32(org.apache.tuweni.bytes.Bytes32) HashMap(java.util.HashMap) Map(java.util.Map)

Example 4 with SnapDataRequest

use of org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest in project besu by hyperledger.

the class SnapDownloaderFactory method createSnapDownloader.

public static Optional<FastSyncDownloader<?>> createSnapDownloader(final PivotBlockSelector pivotBlockSelector, final SynchronizerConfiguration syncConfig, final Path dataDirectory, final ProtocolSchedule protocolSchedule, final ProtocolContext protocolContext, final MetricsSystem metricsSystem, final EthContext ethContext, final WorldStateStorage worldStateStorage, final SyncState syncState, final Clock clock) {
    final Path fastSyncDataDirectory = dataDirectory.resolve(FAST_SYNC_FOLDER);
    final FastSyncStateStorage fastSyncStateStorage = new FastSyncStateStorage(fastSyncDataDirectory);
    if (SyncMode.isFullSync(syncConfig.getSyncMode())) {
        if (fastSyncStateStorage.isFastSyncInProgress()) {
            throw new IllegalStateException("Unable to change the sync mode when snap sync is incomplete, please restart with snap sync mode");
        } else {
            return Optional.empty();
        }
    }
    ensureDirectoryExists(fastSyncDataDirectory.toFile());
    final FastSyncState fastSyncState = fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule));
    if (fastSyncState.getPivotBlockHeader().isEmpty() && protocolContext.getBlockchain().getChainHeadBlockNumber() != BlockHeader.GENESIS_BLOCK_NUMBER) {
        LOG.info("Snap sync was requested, but cannot be enabled because the local blockchain is not empty.");
        return Optional.empty();
    }
    final SnapSyncState snapSyncState = new SnapSyncState(fastSyncStateStorage.loadState(ScheduleBasedBlockHeaderFunctions.create(protocolSchedule)));
    worldStateStorage.clear();
    final InMemoryTasksPriorityQueues<SnapDataRequest> snapTaskCollection = createSnapWorldStateDownloaderTaskCollection();
    final WorldStateDownloader snapWorldStateDownloader = new SnapWorldStateDownloader(ethContext, worldStateStorage, snapTaskCollection, syncConfig.getSnapSyncConfiguration(), syncConfig.getWorldStateRequestParallelism(), syncConfig.getWorldStateMaxRequestsWithoutProgress(), syncConfig.getWorldStateMinMillisBeforeStalling(), clock, metricsSystem);
    final FastSyncDownloader<SnapDataRequest> fastSyncDownloader = new SnapSyncDownloader(new FastSyncActions(syncConfig, worldStateStorage, protocolSchedule, protocolContext, ethContext, syncState, pivotBlockSelector, metricsSystem), worldStateStorage, snapWorldStateDownloader, fastSyncStateStorage, snapTaskCollection, fastSyncDataDirectory, snapSyncState);
    syncState.setWorldStateDownloadStatus(snapWorldStateDownloader);
    return Optional.of(fastSyncDownloader);
}
Also used : Path(java.nio.file.Path) FastSyncState(org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState) FastSyncActions(org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncActions) SnapDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest) FastSyncStateStorage(org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncStateStorage) WorldStateDownloader(org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader)

Example 5 with SnapDataRequest

use of org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest in project besu by hyperledger.

the class RequestDataStep method requestTrieNodeByPath.

public CompletableFuture<List<Task<SnapDataRequest>>> requestTrieNodeByPath(final List<Task<SnapDataRequest>> requestTasks) {
    final BlockHeader blockHeader = fastSyncState.getPivotBlockHeader().get();
    final Map<Bytes, List<Bytes>> message = new HashMap<>();
    requestTasks.stream().map(Task::getData).map(TrieNodeDataRequest.class::cast).map(TrieNodeDataRequest::getTrieNodePath).forEach(path -> {
        final List<Bytes> bytes = message.computeIfAbsent(path.get(0), k -> Lists.newArrayList());
        if (path.size() > 1) {
            bytes.add(path.get(1));
        }
    });
    final EthTask<Map<Bytes, Bytes>> getTrieNodeFromPeerTask = RetryingGetTrieNodeFromPeerTask.forTrieNodes(ethContext, message, blockHeader, metricsSystem);
    downloadState.addOutstandingTask(getTrieNodeFromPeerTask);
    return getTrieNodeFromPeerTask.run().handle((response, error) -> {
        if (response != null) {
            downloadState.removeOutstandingTask(getTrieNodeFromPeerTask);
            for (final Task<SnapDataRequest> task : requestTasks) {
                final TrieNodeDataRequest request = (TrieNodeDataRequest) task.getData();
                final Bytes matchingData = response.get(request.getPathId());
                if (matchingData != null) {
                    request.setData(matchingData);
                }
            }
        }
        return requestTasks;
    });
}
Also used : Bytes(org.apache.tuweni.bytes.Bytes) HashMap(java.util.HashMap) SnapDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest) TrieNodeDataRequest(org.hyperledger.besu.ethereum.eth.sync.snapsync.request.TrieNodeDataRequest) List(java.util.List) BlockHeader(org.hyperledger.besu.ethereum.core.BlockHeader) HashMap(java.util.HashMap) Map(java.util.Map)

Aggregations

SnapDataRequest (org.hyperledger.besu.ethereum.eth.sync.snapsync.request.SnapDataRequest)6 List (java.util.List)3 Bytes (org.apache.tuweni.bytes.Bytes)3 BytecodeRequest (org.hyperledger.besu.ethereum.eth.sync.snapsync.request.BytecodeRequest)3 Task (org.hyperledger.besu.services.tasks.Task)3 Path (java.nio.file.Path)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 Bytes32 (org.apache.tuweni.bytes.Bytes32)2 Hash (org.hyperledger.besu.datatypes.Hash)2 BlockHeader (org.hyperledger.besu.ethereum.core.BlockHeader)2 InMemoryKeyValueStorageProvider (org.hyperledger.besu.ethereum.core.InMemoryKeyValueStorageProvider)2 FastSyncActions (org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncActions)2 FastSyncState (org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncState)2 FastSyncStateStorage (org.hyperledger.besu.ethereum.eth.sync.fastsync.FastSyncStateStorage)2 AccountRangeDataRequest (org.hyperledger.besu.ethereum.eth.sync.snapsync.request.AccountRangeDataRequest)2 StorageRangeDataRequest (org.hyperledger.besu.ethereum.eth.sync.snapsync.request.StorageRangeDataRequest)2 WorldStateDownloader (org.hyperledger.besu.ethereum.eth.sync.worldstate.WorldStateDownloader)2 StoredMerklePatriciaTrie (org.hyperledger.besu.ethereum.trie.StoredMerklePatriciaTrie)2 DataStorageFormat (org.hyperledger.besu.ethereum.worldstate.DataStorageFormat)2