Search in sources :

Example 1 with RemoteSplit

use of com.facebook.presto.split.RemoteSplit in project presto by prestodb.

the class ExchangeOperator method addSplit.

@Override
public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) {
    requireNonNull(split, "split is null");
    checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split");
    RemoteSplit remoteSplit = (RemoteSplit) split.getConnectorSplit();
    exchangeClient.addLocation(remoteSplit.getLocation().toURI(), remoteSplit.getRemoteSourceTaskId());
    return Optional::empty;
}
Also used : RemoteSplit(com.facebook.presto.split.RemoteSplit)

Example 2 with RemoteSplit

use of com.facebook.presto.split.RemoteSplit in project presto by prestodb.

the class MergeOperator method addSplit.

@Override
public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) {
    requireNonNull(split, "split is null");
    checkArgument(split.getConnectorSplit() instanceof RemoteSplit, "split is not a remote split");
    checkState(!blockedOnSplits.isDone(), "noMoreSplits has been called already");
    RemoteSplit remoteSplit = (RemoteSplit) split.getConnectorSplit();
    ExchangeClient exchangeClient = closer.register(taskExchangeClientManager.createExchangeClient(operatorContext.localSystemMemoryContext()));
    exchangeClient.addLocation(remoteSplit.getLocation().toURI(), remoteSplit.getRemoteSourceTaskId());
    exchangeClient.noMoreLocations();
    pageProducers.add(exchangeClient.pages().map(serializedPage -> {
        operatorContext.recordRawInput(serializedPage.getSizeInBytes(), serializedPage.getPositionCount());
        return pagesSerde.deserialize(serializedPage);
    }));
    return Optional::empty;
}
Also used : UpdatablePageSource(com.facebook.presto.spi.UpdatablePageSource) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) Page(com.facebook.presto.common.Page) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) PagesSerdeFactory(com.facebook.presto.execution.buffer.PagesSerdeFactory) SettableFuture(com.google.common.util.concurrent.SettableFuture) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) MoreLists.mappedCopy(com.facebook.presto.util.MoreLists.mappedCopy) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) MergeSortedPages.mergeSortedPages(com.facebook.presto.util.MergeSortedPages.mergeSortedPages) OrderingCompiler(com.facebook.presto.sql.gen.OrderingCompiler) Closer(com.google.common.io.Closer) Objects.requireNonNull(java.util.Objects.requireNonNull) RemoteSplit(com.facebook.presto.split.RemoteSplit) Type(com.facebook.presto.common.type.Type) SortOrder(com.facebook.presto.common.block.SortOrder) IOException(java.io.IOException) Preconditions.checkState(com.google.common.base.Preconditions.checkState) UncheckedIOException(java.io.UncheckedIOException) List(java.util.List) PagesSerde(com.facebook.presto.spi.page.PagesSerde) Closeable(java.io.Closeable) Split(com.facebook.presto.metadata.Split) Optional(java.util.Optional) RemoteSplit(com.facebook.presto.split.RemoteSplit)

Example 3 with RemoteSplit

use of com.facebook.presto.split.RemoteSplit in project presto by prestodb.

the class Driver method processNewSources.

@GuardedBy("exclusiveLock")
private void processNewSources() {
    checkLockHeld("Lock must be held to call processNewSources");
    // only update if the driver is still alive
    if (state.get() != State.ALIVE) {
        return;
    }
    TaskSource sourceUpdate = pendingTaskSourceUpdates.getAndSet(null);
    if (sourceUpdate == null) {
        return;
    }
    // merge the current source and the specified source update
    TaskSource newSource = currentTaskSource.update(sourceUpdate);
    // if the update contains no new data, just return
    if (newSource == currentTaskSource) {
        return;
    }
    // determine new splits to add
    Set<ScheduledSplit> newSplits = Sets.difference(newSource.getSplits(), currentTaskSource.getSplits());
    // add new splits
    SourceOperator sourceOperator = this.sourceOperator.orElseThrow(VerifyException::new);
    for (ScheduledSplit newSplit : newSplits) {
        Split split = newSplit.getSplit();
        if (fragmentResultCacheContext.get().isPresent() && !(split.getConnectorSplit() instanceof RemoteSplit)) {
            checkState(!this.cachedResult.get().isPresent());
            this.fragmentResultCacheContext.set(this.fragmentResultCacheContext.get().map(context -> context.updateRuntimeInformation(split.getConnectorSplit())));
            Optional<Iterator<Page>> pages = fragmentResultCacheContext.get().get().getFragmentResultCacheManager().get(fragmentResultCacheContext.get().get().getHashedCanonicalPlanFragment(), split);
            sourceOperator.getOperatorContext().getRuntimeStats().addMetricValue(pages.isPresent() ? RuntimeMetricName.FRAGMENT_RESULT_CACHE_HIT : RuntimeMetricName.FRAGMENT_RESULT_CACHE_MISS, 1);
            this.cachedResult.set(pages);
            this.split.set(split);
        }
        Supplier<Optional<UpdatablePageSource>> pageSource = sourceOperator.addSplit(split);
        deleteOperator.ifPresent(deleteOperator -> deleteOperator.setPageSource(pageSource));
    }
    // set no more splits
    if (newSource.isNoMoreSplits()) {
        sourceOperator.noMoreSplits();
    }
    currentTaskSource = newSource;
}
Also used : UpdatablePageSource(com.facebook.presto.spi.UpdatablePageSource) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) Page(com.facebook.presto.common.Page) Logger(com.facebook.airlift.log.Logger) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) NOT_BLOCKED(com.facebook.presto.operator.Operator.NOT_BLOCKED) NO_PREFERENCE(com.facebook.presto.spi.schedule.NodeSelectionStrategy.NO_PREFERENCE) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) Throwables.throwIfUnchecked(com.google.common.base.Throwables.throwIfUnchecked) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) SettableFuture(com.google.common.util.concurrent.SettableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Supplier(java.util.function.Supplier) Duration(io.airlift.units.Duration) ArrayList(java.util.ArrayList) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) RemoteSplit(com.facebook.presto.split.RemoteSplit) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) VerifyException(com.google.common.base.VerifyException) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) RuntimeMetricName(com.facebook.presto.common.RuntimeMetricName) TaskSource(com.facebook.presto.execution.TaskSource) Set(java.util.Set) GuardedBy(javax.annotation.concurrent.GuardedBy) Sets(com.google.common.collect.Sets) Preconditions.checkState(com.google.common.base.Preconditions.checkState) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) TimeUnit(java.util.concurrent.TimeUnit) FragmentResultCacheContext(com.facebook.presto.execution.FragmentResultCacheContext) List(java.util.List) MoreUninterruptibles.tryLockUninterruptibly(com.facebook.presto.util.MoreUninterruptibles.tryLockUninterruptibly) Closeable(java.io.Closeable) Split(com.facebook.presto.metadata.Split) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TRUE(java.lang.Boolean.TRUE) SpillingUtils.checkSpillSucceeded(com.facebook.presto.operator.SpillingUtils.checkSpillSucceeded) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) Optional(java.util.Optional) VerifyException(com.google.common.base.VerifyException) RemoteSplit(com.facebook.presto.split.RemoteSplit) Iterator(java.util.Iterator) RemoteSplit(com.facebook.presto.split.RemoteSplit) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) Split(com.facebook.presto.metadata.Split) TaskSource(com.facebook.presto.execution.TaskSource) GuardedBy(javax.annotation.concurrent.GuardedBy)

Aggregations

RemoteSplit (com.facebook.presto.split.RemoteSplit)3 Page (com.facebook.presto.common.Page)2 Split (com.facebook.presto.metadata.Split)2 UpdatablePageSource (com.facebook.presto.spi.UpdatablePageSource)2 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 Preconditions.checkState (com.google.common.base.Preconditions.checkState)2 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)2 SettableFuture (com.google.common.util.concurrent.SettableFuture)2 Closeable (java.io.Closeable)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Objects.requireNonNull (java.util.Objects.requireNonNull)2 Optional (java.util.Optional)2 Supplier (java.util.function.Supplier)2 Logger (com.facebook.airlift.log.Logger)1 RuntimeMetricName (com.facebook.presto.common.RuntimeMetricName)1 SortOrder (com.facebook.presto.common.block.SortOrder)1 Type (com.facebook.presto.common.type.Type)1 FragmentResultCacheContext (com.facebook.presto.execution.FragmentResultCacheContext)1