Search in sources :

Example 11 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class HttpRemoteTask method processTaskUpdate.

private synchronized void processTaskUpdate(TaskInfo newValue, List<TaskSource> sources) {
    updateTaskInfo(newValue);
    // remove acknowledged splits, which frees memory
    for (TaskSource source : sources) {
        PlanNodeId planNodeId = source.getPlanNodeId();
        boolean isTableScanSource = tableScanPlanNodeIds.contains(planNodeId);
        int removed = 0;
        long removedWeight = 0;
        for (ScheduledSplit split : source.getSplits()) {
            if (pendingSplits.remove(planNodeId, split)) {
                if (isTableScanSource) {
                    removed++;
                    removedWeight = addExact(removedWeight, split.getSplit().getSplitWeight().getRawValue());
                }
            }
        }
        if (source.isNoMoreSplits()) {
            noMoreSplits.put(planNodeId, false);
        }
        for (Lifespan lifespan : source.getNoMoreSplitsForLifespan()) {
            pendingNoMoreSplitsForLifespan.remove(planNodeId, lifespan);
        }
        if (isTableScanSource) {
            pendingSourceSplitCount -= removed;
            pendingSourceSplitsWeight -= removedWeight;
        }
    }
    // Update stats before split queue space to ensure node stats are up to date before waking up the scheduler
    updateTaskStats();
    updateSplitQueueSpace();
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) Lifespan(com.facebook.presto.execution.Lifespan) TaskSource(com.facebook.presto.execution.TaskSource)

Example 12 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class TestSystemMemoryBlocking method testTableScanSystemMemoryBlocking.

@Test
public void testTableScanSystemMemoryBlocking() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).build()), new TableHandle(new ConnectorId("test"), new ConnectorTableHandle() {
    }, new ConnectorTransactionHandle() {
    }, Optional.empty()), ImmutableList.of());
    PageConsumerOperator sink = createSinkOperator(types);
    Driver driver = Driver.createDriver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    Split testSplit = new Split(new ConnectorId("test"), TestingTransactionHandle.create(), new TestSplit());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, NANOSECONDS));
    // the driver shouldn't block in the first call as it will be able to move a page between source and the sink operator
    // but the operator should be blocked
    assertTrue(blocked.isDone());
    assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    // and they should stay blocked until more memory becomes available
    for (int i = 0; i < 10; i++) {
        blocked = driver.processFor(new Duration(1, NANOSECONDS));
        assertFalse(blocked.isDone());
        assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    }
    // free up some memory
    memoryPool.free(QUERY_ID, "test", memoryPool.getReservedBytes());
    // the operator should be unblocked
    assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
    // the driver shouldn't be blocked
    blocked = driver.processFor(new Duration(1, NANOSECONDS));
    assertTrue(blocked.isDone());
}
Also used : TableScanOperator(com.facebook.presto.operator.TableScanOperator) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) Driver(com.facebook.presto.operator.Driver) Duration(io.airlift.units.Duration) FixedPageSource(com.facebook.presto.spi.FixedPageSource) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) PageConsumerOperator(com.facebook.presto.testing.PageConsumerOperator) Type(com.facebook.presto.common.type.Type) ConnectorTableHandle(com.facebook.presto.spi.ConnectorTableHandle) TableHandle(com.facebook.presto.spi.TableHandle) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Split(com.facebook.presto.metadata.Split) TaskSource(com.facebook.presto.execution.TaskSource) ConnectorId(com.facebook.presto.spi.ConnectorId) Test(org.testng.annotations.Test)

Example 13 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class Driver method processNewSources.

@GuardedBy("exclusiveLock")
private void processNewSources() {
    checkLockHeld("Lock must be held to call processNewSources");
    // only update if the driver is still alive
    if (state.get() != State.ALIVE) {
        return;
    }
    TaskSource sourceUpdate = pendingTaskSourceUpdates.getAndSet(null);
    if (sourceUpdate == null) {
        return;
    }
    // merge the current source and the specified source update
    TaskSource newSource = currentTaskSource.update(sourceUpdate);
    // if the update contains no new data, just return
    if (newSource == currentTaskSource) {
        return;
    }
    // determine new splits to add
    Set<ScheduledSplit> newSplits = Sets.difference(newSource.getSplits(), currentTaskSource.getSplits());
    // add new splits
    SourceOperator sourceOperator = this.sourceOperator.orElseThrow(VerifyException::new);
    for (ScheduledSplit newSplit : newSplits) {
        Split split = newSplit.getSplit();
        if (fragmentResultCacheContext.get().isPresent() && !(split.getConnectorSplit() instanceof RemoteSplit)) {
            checkState(!this.cachedResult.get().isPresent());
            this.fragmentResultCacheContext.set(this.fragmentResultCacheContext.get().map(context -> context.updateRuntimeInformation(split.getConnectorSplit())));
            Optional<Iterator<Page>> pages = fragmentResultCacheContext.get().get().getFragmentResultCacheManager().get(fragmentResultCacheContext.get().get().getHashedCanonicalPlanFragment(), split);
            sourceOperator.getOperatorContext().getRuntimeStats().addMetricValue(pages.isPresent() ? RuntimeMetricName.FRAGMENT_RESULT_CACHE_HIT : RuntimeMetricName.FRAGMENT_RESULT_CACHE_MISS, 1);
            this.cachedResult.set(pages);
            this.split.set(split);
        }
        Supplier<Optional<UpdatablePageSource>> pageSource = sourceOperator.addSplit(split);
        deleteOperator.ifPresent(deleteOperator -> deleteOperator.setPageSource(pageSource));
    }
    // set no more splits
    if (newSource.isNoMoreSplits()) {
        sourceOperator.noMoreSplits();
    }
    currentTaskSource = newSource;
}
Also used : UpdatablePageSource(com.facebook.presto.spi.UpdatablePageSource) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) Page(com.facebook.presto.common.Page) Logger(com.facebook.airlift.log.Logger) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) NOT_BLOCKED(com.facebook.presto.operator.Operator.NOT_BLOCKED) NO_PREFERENCE(com.facebook.presto.spi.schedule.NodeSelectionStrategy.NO_PREFERENCE) GENERIC_INTERNAL_ERROR(com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) Throwables.throwIfUnchecked(com.google.common.base.Throwables.throwIfUnchecked) HashMap(java.util.HashMap) PrestoException(com.facebook.presto.spi.PrestoException) SettableFuture(com.google.common.util.concurrent.SettableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) Supplier(java.util.function.Supplier) Duration(io.airlift.units.Duration) ArrayList(java.util.ArrayList) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) RemoteSplit(com.facebook.presto.split.RemoteSplit) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) VerifyException(com.google.common.base.VerifyException) ImmutableSet(com.google.common.collect.ImmutableSet) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) RuntimeMetricName(com.facebook.presto.common.RuntimeMetricName) TaskSource(com.facebook.presto.execution.TaskSource) Set(java.util.Set) GuardedBy(javax.annotation.concurrent.GuardedBy) Sets(com.google.common.collect.Sets) Preconditions.checkState(com.google.common.base.Preconditions.checkState) MoreExecutors.directExecutor(com.google.common.util.concurrent.MoreExecutors.directExecutor) TimeUnit(java.util.concurrent.TimeUnit) FragmentResultCacheContext(com.facebook.presto.execution.FragmentResultCacheContext) List(java.util.List) MoreUninterruptibles.tryLockUninterruptibly(com.facebook.presto.util.MoreUninterruptibles.tryLockUninterruptibly) Closeable(java.io.Closeable) Split(com.facebook.presto.metadata.Split) Optional(java.util.Optional) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TRUE(java.lang.Boolean.TRUE) SpillingUtils.checkSpillSucceeded(com.facebook.presto.operator.SpillingUtils.checkSpillSucceeded) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) Optional(java.util.Optional) VerifyException(com.google.common.base.VerifyException) RemoteSplit(com.facebook.presto.split.RemoteSplit) Iterator(java.util.Iterator) RemoteSplit(com.facebook.presto.split.RemoteSplit) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) Split(com.facebook.presto.metadata.Split) TaskSource(com.facebook.presto.execution.TaskSource) GuardedBy(javax.annotation.concurrent.GuardedBy)

Example 14 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class IndexLoader method streamIndexDataForSingleKey.

public IndexedData streamIndexDataForSingleKey(UpdateRequest updateRequest) {
    Page indexKeyTuple = updateRequest.getPage().getRegion(0, 1);
    PageBuffer pageBuffer = new PageBuffer(100);
    DriverFactory driverFactory = indexBuildDriverFactoryProvider.createStreaming(pageBuffer, indexKeyTuple);
    Driver driver = driverFactory.createDriver(pipelineContext.addDriverContext());
    PageRecordSet pageRecordSet = new PageRecordSet(keyTypes, indexKeyTuple);
    PlanNodeId planNodeId = driverFactory.getSourceId().get();
    ScheduledSplit split = new ScheduledSplit(0, planNodeId, new Split(INDEX_CONNECTOR_ID, new ConnectorTransactionHandle() {
    }, new IndexSplit(pageRecordSet)));
    driver.updateSource(new TaskSource(planNodeId, ImmutableSet.of(split), true));
    return new StreamingIndexedData(outputTypes, keyTypes, indexKeyTuple, pageBuffer, driver);
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) DriverFactory(com.facebook.presto.operator.DriverFactory) ConnectorTransactionHandle(com.facebook.presto.spi.connector.ConnectorTransactionHandle) Driver(com.facebook.presto.operator.Driver) Page(com.facebook.presto.common.Page) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) Split(com.facebook.presto.metadata.Split) TaskSource(com.facebook.presto.execution.TaskSource)

Example 15 with TaskSource

use of com.facebook.presto.execution.TaskSource in project presto by prestodb.

the class PrestoSparkTaskExecution method scheduleDriversForSplitLifeCycle.

private synchronized void scheduleDriversForSplitLifeCycle(List<TaskSource> sources) {
    checkArgument(sources.stream().allMatch(TaskSource::isNoMoreSplits), "All task sources are expected to be final");
    ListMultimap<PlanNodeId, ScheduledSplit> splits = ArrayListMultimap.create();
    for (TaskSource taskSource : sources) {
        splits.putAll(taskSource.getPlanNodeId(), taskSource.getSplits());
    }
    for (PlanNodeId planNodeId : schedulingOrder) {
        DriverSplitRunnerFactory driverSplitRunnerFactory = driverRunnerFactoriesWithSplitLifeCycle.get(planNodeId);
        List<ScheduledSplit> planNodeSplits = splits.get(planNodeId);
        scheduleTableScanSource(driverSplitRunnerFactory, planNodeSplits);
    }
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) ScheduledSplit(com.facebook.presto.execution.ScheduledSplit) TaskSource(com.facebook.presto.execution.TaskSource)

Aggregations

TaskSource (com.facebook.presto.execution.TaskSource)16 ScheduledSplit (com.facebook.presto.execution.ScheduledSplit)12 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)10 Duration (io.airlift.units.Duration)5 Type (com.facebook.presto.common.type.Type)4 SerializedPrestoSparkTaskSource (com.facebook.presto.spark.classloader_interface.SerializedPrestoSparkTaskSource)4 FixedPageSource (com.facebook.presto.spi.FixedPageSource)4 Split (com.facebook.presto.metadata.Split)3 PageConsumerOperator (com.facebook.presto.testing.PageConsumerOperator)3 ImmutableList (com.google.common.collect.ImmutableList)3 JsonCodec (com.facebook.airlift.json.JsonCodec)2 SmileCodec (com.facebook.airlift.json.smile.SmileCodec)2 Page (com.facebook.presto.common.Page)2 TaskManagerConfig (com.facebook.presto.execution.TaskManagerConfig)2 MemoryManagerConfig (com.facebook.presto.memory.MemoryManagerConfig)2 Driver (com.facebook.presto.operator.Driver)2 LookupJoinOperators (com.facebook.presto.operator.LookupJoinOperators)2 NoOpFragmentResultCacheManager (com.facebook.presto.operator.NoOpFragmentResultCacheManager)2 PagesIndex (com.facebook.presto.operator.PagesIndex)2 TableCommitContext (com.facebook.presto.operator.TableCommitContext)2