Search in sources :

Example 16 with DriverContext

use of com.facebook.presto.operator.DriverContext in project presto by prestodb.

the class HashJoinBenchmark method createDrivers.

/*
    select orderkey, quantity, totalprice
    from lineitem join orders using (orderkey)
     */
@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    if (probeDriverFactory == null) {
        List<Type> ordersTypes = getColumnTypes("orders", "orderkey", "totalprice");
        OperatorFactory ordersTableScan = createTableScanOperator(0, new PlanNodeId("test"), "orders", "orderkey", "totalprice");
        JoinBridgeManager<PartitionedLookupSourceFactory> lookupSourceFactoryManager = JoinBridgeManager.lookupAllAtOnce(new PartitionedLookupSourceFactory(ordersTypes, ImmutableList.of(0, 1).stream().map(ordersTypes::get).collect(toImmutableList()), Ints.asList(0).stream().map(ordersTypes::get).collect(toImmutableList()), 1, requireNonNull(ImmutableMap.of(), "layout is null"), false));
        HashBuilderOperatorFactory hashBuilder = new HashBuilderOperatorFactory(1, new PlanNodeId("test"), lookupSourceFactoryManager, ImmutableList.of(0, 1), Ints.asList(0), OptionalInt.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(), 1_500_000, new PagesIndex.TestingFactory(false), false, SingleStreamSpillerFactory.unsupportedSingleStreamSpillerFactory(), false);
        DriverContext driverContext = taskContext.addPipelineContext(0, false, false, false).addDriverContext();
        DriverFactory buildDriverFactory = new DriverFactory(0, false, false, ImmutableList.of(ordersTableScan, hashBuilder), OptionalInt.empty(), UNGROUPED_EXECUTION, Optional.empty());
        List<Type> lineItemTypes = getColumnTypes("lineitem", "orderkey", "quantity");
        OperatorFactory lineItemTableScan = createTableScanOperator(0, new PlanNodeId("test"), "lineitem", "orderkey", "quantity");
        OperatorFactory joinOperator = LOOKUP_JOIN_OPERATORS.innerJoin(1, new PlanNodeId("test"), lookupSourceFactoryManager, lineItemTypes, Ints.asList(0), OptionalInt.empty(), Optional.empty(), OptionalInt.empty(), unsupportedPartitioningSpillerFactory());
        NullOutputOperatorFactory output = new NullOutputOperatorFactory(2, new PlanNodeId("test"));
        this.probeDriverFactory = new DriverFactory(1, true, true, ImmutableList.of(lineItemTableScan, joinOperator, output), OptionalInt.empty(), UNGROUPED_EXECUTION, Optional.empty());
        Driver driver = buildDriverFactory.createDriver(driverContext);
        Future<LookupSourceProvider> lookupSourceProvider = lookupSourceFactoryManager.getJoinBridge(Lifespan.taskWide()).createLookupSourceProvider();
        while (!lookupSourceProvider.isDone()) {
            driver.process();
        }
        getFutureValue(lookupSourceProvider).close();
    }
    DriverContext driverContext = taskContext.addPipelineContext(1, true, true, false).addDriverContext();
    Driver driver = probeDriverFactory.createDriver(driverContext);
    return ImmutableList.of(driver);
}
Also used : DriverContext(com.facebook.presto.operator.DriverContext) HashBuilderOperatorFactory(com.facebook.presto.operator.HashBuilderOperator.HashBuilderOperatorFactory) Driver(com.facebook.presto.operator.Driver) PagesIndex(com.facebook.presto.operator.PagesIndex) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) Type(com.facebook.presto.common.type.Type) OperatorFactory(com.facebook.presto.operator.OperatorFactory) HashBuilderOperatorFactory(com.facebook.presto.operator.HashBuilderOperator.HashBuilderOperatorFactory) NullOutputOperatorFactory(com.facebook.presto.testing.NullOutputOperator.NullOutputOperatorFactory) DriverFactory(com.facebook.presto.operator.DriverFactory) PartitionedLookupSourceFactory(com.facebook.presto.operator.PartitionedLookupSourceFactory) NullOutputOperatorFactory(com.facebook.presto.testing.NullOutputOperator.NullOutputOperatorFactory) LookupSourceProvider(com.facebook.presto.operator.LookupSourceProvider)

Example 17 with DriverContext

use of com.facebook.presto.operator.DriverContext in project presto by prestodb.

the class AbstractSimpleOperatorBenchmark method createDrivers.

@Override
protected List<Driver> createDrivers(TaskContext taskContext) {
    DriverFactory driverFactory = createDriverFactory();
    DriverContext driverContext = taskContext.addPipelineContext(0, true, true, false).addDriverContext();
    Driver driver = driverFactory.createDriver(driverContext);
    return ImmutableList.of(driver);
}
Also used : DriverContext(com.facebook.presto.operator.DriverContext) DriverFactory(com.facebook.presto.operator.DriverFactory) Driver(com.facebook.presto.operator.Driver)

Example 18 with DriverContext

use of com.facebook.presto.operator.DriverContext in project presto by prestodb.

the class SqlTaskExecution method enqueueDriverSplitRunner.

private synchronized void enqueueDriverSplitRunner(boolean forceRunSplit, List<DriverSplitRunner> runners) {
    // schedule driver to be executed
    List<ListenableFuture<?>> finishedFutures = taskExecutor.enqueueSplits(taskHandle, forceRunSplit, runners);
    checkState(finishedFutures.size() == runners.size(), "Expected %s futures but got %s", runners.size(), finishedFutures.size());
    // when driver completes, update state and fire events
    for (int i = 0; i < finishedFutures.size(); i++) {
        ListenableFuture<?> finishedFuture = finishedFutures.get(i);
        final DriverSplitRunner splitRunner = runners.get(i);
        // record new driver
        status.incrementRemainingDriver(splitRunner.getLifespan());
        Futures.addCallback(finishedFuture, new FutureCallback<Object>() {

            @Override
            public void onSuccess(Object result) {
                try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) {
                    // record driver is finished
                    status.decrementRemainingDriver(splitRunner.getLifespan());
                    checkTaskCompletion();
                    splitMonitor.splitCompletedEvent(taskId, getDriverStats());
                }
            }

            @Override
            public void onFailure(Throwable cause) {
                try (SetThreadName ignored = new SetThreadName("Task-%s", taskId)) {
                    taskStateMachine.failed(cause);
                    // record driver is finished
                    status.decrementRemainingDriver(splitRunner.getLifespan());
                    // fire failed event with cause
                    splitMonitor.splitFailedEvent(taskId, getDriverStats(), cause);
                }
            }

            private DriverStats getDriverStats() {
                DriverContext driverContext = splitRunner.getDriverContext();
                DriverStats driverStats;
                if (driverContext != null) {
                    driverStats = driverContext.getDriverStats();
                } else {
                    // split runner did not start successfully
                    driverStats = new DriverStats();
                }
                return driverStats;
            }
        }, notificationExecutor);
    }
}
Also used : DriverContext(com.facebook.presto.operator.DriverContext) DriverStats(com.facebook.presto.operator.DriverStats) SetThreadName(com.facebook.airlift.concurrent.SetThreadName) ListenableFuture(com.google.common.util.concurrent.ListenableFuture)

Example 19 with DriverContext

use of com.facebook.presto.operator.DriverContext in project presto by prestodb.

the class TestMemoryRevokingScheduler method createContexts.

private OperatorContext createContexts(SqlTask sqlTask) {
    TaskContext taskContext = getOrCreateTaskContext(sqlTask);
    PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false);
    DriverContext driverContext = pipelineContext.addDriverContext();
    return driverContext.addOperatorContext(1, new PlanNodeId("na"), "na");
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) DriverContext(com.facebook.presto.operator.DriverContext) TaskContext(com.facebook.presto.operator.TaskContext) PipelineContext(com.facebook.presto.operator.PipelineContext)

Example 20 with DriverContext

use of com.facebook.presto.operator.DriverContext in project presto by prestodb.

the class TestMemoryRevokingScheduler method createTestingOperatorContexts.

private TestOperatorContext createTestingOperatorContexts(SqlTask sqlTask, String operatorName) {
    // update task to update underlying taskHolderReference with taskExecution + create a new taskContext
    sqlTask.updateTask(TEST_SESSION, Optional.of(PLAN_FRAGMENT), ImmutableList.of(new TaskSource(TABLE_SCAN_NODE_ID, ImmutableSet.of(SPLIT), false)), createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds(), Optional.of(new TableWriteInfo(Optional.empty(), Optional.empty(), Optional.empty())));
    // use implicitly created task context from updateTask. It should be the only task in this QueryContext's tasks
    TaskContext taskContext = sqlTask.getQueryContext().getTaskContextByTaskId(sqlTask.getTaskId());
    PipelineContext pipelineContext = taskContext.addPipelineContext(0, false, false, false);
    DriverContext driverContext = pipelineContext.addDriverContext();
    TestOperatorContext testOperatorContext = new TestOperatorContext(1, new PlanNodeId("na"), "na", driverContext, singleThreadedExecutor, driverContext.getDriverMemoryContext().newMemoryTrackingContext(), operatorName);
    driverContext.addOperatorContext(testOperatorContext);
    return testOperatorContext;
}
Also used : PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) DriverContext(com.facebook.presto.operator.DriverContext) TableWriteInfo(com.facebook.presto.execution.scheduler.TableWriteInfo) TaskContext(com.facebook.presto.operator.TaskContext) PipelineContext(com.facebook.presto.operator.PipelineContext)

Aggregations

DriverContext (com.facebook.presto.operator.DriverContext)24 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)13 Test (org.testng.annotations.Test)12 TaskContext (com.facebook.presto.operator.TaskContext)10 OperatorFactory (com.facebook.presto.operator.OperatorFactory)8 Driver (com.facebook.presto.operator.Driver)6 OperatorContext (com.facebook.presto.operator.OperatorContext)6 PagesSpatialIndexFactory (com.facebook.presto.operator.PagesSpatialIndexFactory)5 SpatialIndexBuilderOperatorFactory (com.facebook.presto.operator.SpatialIndexBuilderOperator.SpatialIndexBuilderOperatorFactory)5 SpatialJoinOperatorFactory (com.facebook.presto.operator.SpatialJoinOperator.SpatialJoinOperatorFactory)5 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)5 RowPagesBuilder (com.facebook.presto.RowPagesBuilder)4 Page (com.facebook.presto.common.Page)4 LocalMemoryContext (com.facebook.presto.memory.context.LocalMemoryContext)4 Split (com.facebook.presto.metadata.Split)4 DriverFactory (com.facebook.presto.operator.DriverFactory)4 PipelineContext (com.facebook.presto.operator.PipelineContext)4 QueryId (com.facebook.presto.spi.QueryId)4 MaterializedResult (com.facebook.presto.testing.MaterializedResult)4 TestingTaskContext (com.facebook.presto.testing.TestingTaskContext)4