use of com.facebook.presto.split.RemoteSplit in project presto by prestodb.
the class ExchangeOperator method addSplit.
@Override
public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) {
requireNonNull(split, "split is null");
checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split");
RemoteSplit remoteSplit = (RemoteSplit) split.getConnectorSplit();
exchangeClient.addLocation(remoteSplit.getLocation().toURI(), remoteSplit.getRemoteSourceTaskId());
return Optional::empty;
}
use of com.facebook.presto.split.RemoteSplit in project presto by prestodb.
the class MergeOperator method addSplit.
@Override
public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) {
requireNonNull(split, "split is null");
checkArgument(split.getConnectorSplit() instanceof RemoteSplit, "split is not a remote split");
checkState(!blockedOnSplits.isDone(), "noMoreSplits has been called already");
RemoteSplit remoteSplit = (RemoteSplit) split.getConnectorSplit();
ExchangeClient exchangeClient = closer.register(taskExchangeClientManager.createExchangeClient(operatorContext.localSystemMemoryContext()));
exchangeClient.addLocation(remoteSplit.getLocation().toURI(), remoteSplit.getRemoteSourceTaskId());
exchangeClient.noMoreLocations();
pageProducers.add(exchangeClient.pages().map(serializedPage -> {
operatorContext.recordRawInput(serializedPage.getSizeInBytes(), serializedPage.getPositionCount());
return pagesSerde.deserialize(serializedPage);
}));
return Optional::empty;
}
use of com.facebook.presto.split.RemoteSplit in project presto by prestodb.
the class Driver method processNewSources.
@GuardedBy("exclusiveLock")
private void processNewSources() {
checkLockHeld("Lock must be held to call processNewSources");
// only update if the driver is still alive
if (state.get() != State.ALIVE) {
return;
}
TaskSource sourceUpdate = pendingTaskSourceUpdates.getAndSet(null);
if (sourceUpdate == null) {
return;
}
// merge the current source and the specified source update
TaskSource newSource = currentTaskSource.update(sourceUpdate);
// if the update contains no new data, just return
if (newSource == currentTaskSource) {
return;
}
// determine new splits to add
Set<ScheduledSplit> newSplits = Sets.difference(newSource.getSplits(), currentTaskSource.getSplits());
// add new splits
SourceOperator sourceOperator = this.sourceOperator.orElseThrow(VerifyException::new);
for (ScheduledSplit newSplit : newSplits) {
Split split = newSplit.getSplit();
if (fragmentResultCacheContext.get().isPresent() && !(split.getConnectorSplit() instanceof RemoteSplit)) {
checkState(!this.cachedResult.get().isPresent());
this.fragmentResultCacheContext.set(this.fragmentResultCacheContext.get().map(context -> context.updateRuntimeInformation(split.getConnectorSplit())));
Optional<Iterator<Page>> pages = fragmentResultCacheContext.get().get().getFragmentResultCacheManager().get(fragmentResultCacheContext.get().get().getHashedCanonicalPlanFragment(), split);
sourceOperator.getOperatorContext().getRuntimeStats().addMetricValue(pages.isPresent() ? RuntimeMetricName.FRAGMENT_RESULT_CACHE_HIT : RuntimeMetricName.FRAGMENT_RESULT_CACHE_MISS, 1);
this.cachedResult.set(pages);
this.split.set(split);
}
Supplier<Optional<UpdatablePageSource>> pageSource = sourceOperator.addSplit(split);
deleteOperator.ifPresent(deleteOperator -> deleteOperator.setPageSource(pageSource));
}
// set no more splits
if (newSource.isNoMoreSplits()) {
sourceOperator.noMoreSplits();
}
currentTaskSource = newSource;
}
Aggregations