Search in sources :

Example 1 with ScheduledSplit

use of com.facebook.presto.ScheduledSplit in project presto by prestodb.

the class LocalQueryRunner method createDrivers.

public List<Driver> createDrivers(Session session, @Language("SQL") String sql, OutputFactory outputFactory, TaskContext taskContext) {
    Plan plan = createPlan(session, sql);
    if (printPlan) {
        System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), metadata, session));
    }
    SubPlan subplan = PlanFragmenter.createSubPlans(session, metadata, plan);
    if (!subplan.getChildren().isEmpty()) {
        throw new AssertionError("Expected subplan to have no children");
    }
    LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner(metadata, sqlParser, Optional.empty(), pageSourceManager, indexManager, nodePartitioningManager, pageSinkManager, null, expressionCompiler, joinFilterFunctionCompiler, new IndexJoinLookupStats(), // make sure tests fail if compiler breaks
    new CompilerConfig().setInterpreterEnabled(false), new TaskManagerConfig().setTaskConcurrency(4), spillerFactory, blockEncodingSerde, new PagesIndex.TestingFactory(), new JoinCompiler(), new LookupJoinOperators(new JoinProbeCompiler()));
    // plan query
    LocalExecutionPlan localExecutionPlan = executionPlanner.plan(session, subplan.getFragment().getRoot(), subplan.getFragment().getPartitioningScheme().getOutputLayout(), plan.getTypes(), outputFactory);
    // generate sources
    List<TaskSource> sources = new ArrayList<>();
    long sequenceId = 0;
    for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) {
        TableLayoutHandle layout = tableScan.getLayout().get();
        SplitSource splitSource = splitManager.getSplits(session, layout);
        ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
        while (!splitSource.isFinished()) {
            for (Split split : getFutureValue(splitSource.getNextBatch(1000))) {
                scheduledSplits.add(new ScheduledSplit(sequenceId++, tableScan.getId(), split));
            }
        }
        sources.add(new TaskSource(tableScan.getId(), scheduledSplits.build(), true));
    }
    // create drivers
    List<Driver> drivers = new ArrayList<>();
    Map<PlanNodeId, DriverFactory> driverFactoriesBySource = new HashMap<>();
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        for (int i = 0; i < driverFactory.getDriverInstances().orElse(1); i++) {
            if (driverFactory.getSourceId().isPresent()) {
                checkState(driverFactoriesBySource.put(driverFactory.getSourceId().get(), driverFactory) == null);
            } else {
                DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver()).addDriverContext();
                Driver driver = driverFactory.createDriver(driverContext);
                drivers.add(driver);
            }
        }
    }
    // add sources to the drivers
    for (TaskSource source : sources) {
        DriverFactory driverFactory = driverFactoriesBySource.get(source.getPlanNodeId());
        checkState(driverFactory != null);
        for (ScheduledSplit split : source.getSplits()) {
            DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver()).addDriverContext();
            Driver driver = driverFactory.createDriver(driverContext);
            driver.updateSource(new TaskSource(split.getPlanNodeId(), ImmutableSet.of(split), true));
            drivers.add(driver);
        }
    }
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        driverFactory.close();
    }
    return ImmutableList.copyOf(drivers);
}
Also used : DriverContext(com.facebook.presto.operator.DriverContext) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Driver(com.facebook.presto.operator.Driver) TaskManagerConfig(com.facebook.presto.execution.TaskManagerConfig) PagesIndex(com.facebook.presto.operator.PagesIndex) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) JoinProbeCompiler(com.facebook.presto.sql.gen.JoinProbeCompiler) ImmutableSet(com.google.common.collect.ImmutableSet) DriverFactory(com.facebook.presto.operator.DriverFactory) LookupJoinOperators(com.facebook.presto.operator.LookupJoinOperators) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) ScheduledSplit(com.facebook.presto.ScheduledSplit) LocalExecutionPlanner(com.facebook.presto.sql.planner.LocalExecutionPlanner) IndexJoinLookupStats(com.facebook.presto.operator.index.IndexJoinLookupStats) LocalExecutionPlan(com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) Plan(com.facebook.presto.sql.planner.Plan) SubPlan(com.facebook.presto.sql.planner.SubPlan) CompilerConfig(com.facebook.presto.sql.planner.CompilerConfig) TableLayoutHandle(com.facebook.presto.metadata.TableLayoutHandle) Constraint(com.facebook.presto.spi.Constraint) LocalExecutionPlan(com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) TableScanNode(com.facebook.presto.sql.planner.plan.TableScanNode) SplitSource(com.facebook.presto.split.SplitSource) ScheduledSplit(com.facebook.presto.ScheduledSplit) Split(com.facebook.presto.metadata.Split) SubPlan(com.facebook.presto.sql.planner.SubPlan) TaskSource(com.facebook.presto.TaskSource)

Example 2 with ScheduledSplit

use of com.facebook.presto.ScheduledSplit in project presto by prestodb.

the class TestDriver method testBrokenOperatorAddSource.

@Test
public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    // create a table scan operator that does not block, which will cause the driver loop to busy wait
    TableScanOperator source = new NotBlockedTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, new PageSourceProvider() {

        @Override
        public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) {
            return new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
        }
    }, types, ImmutableList.of());
    BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
    final Driver driver = new Driver(driverContext, source, brokenOperator);
    // block thread in operator processing
    Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
        }
    });
    brokenOperator.waitForLocked();
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.close();
    assertTrue(driver.isFinished());
    try {
        driverProcessFor.get(1, TimeUnit.SECONDS);
        fail("Expected InterruptedException");
    } catch (ExecutionException e) {
        checkArgument(getRootCause(e) instanceof InterruptedException, "Expected root cause exception to be an instance of InterruptedException");
    }
}
Also used : PageSourceProvider(com.facebook.presto.split.PageSourceProvider) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ScheduledSplit(com.facebook.presto.ScheduledSplit) Duration(io.airlift.units.Duration) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) FixedPageSource(com.facebook.presto.spi.FixedPageSource) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) Type(com.facebook.presto.spi.type.Type) ScheduledSplit(com.facebook.presto.ScheduledSplit) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Split(com.facebook.presto.metadata.Split) ExecutionException(java.util.concurrent.ExecutionException) TaskSource(com.facebook.presto.TaskSource) Session(com.facebook.presto.Session) Test(org.testng.annotations.Test)

Example 3 with ScheduledSplit

use of com.facebook.presto.ScheduledSplit in project presto by prestodb.

the class TestDriver method testAddSourceFinish.

@Test
public void testAddSourceFinish() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, new PageSourceProvider() {

        @Override
        public ConnectorPageSource createPageSource(Session session, Split split, List<ColumnHandle> columns) {
            return new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build());
        }
    }, types, ImmutableList.of());
    PageConsumerOperator sink = createSinkOperator(source);
    Driver driver = new Driver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertTrue(driver.isFinished());
    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
}
Also used : PageSourceProvider(com.facebook.presto.split.PageSourceProvider) ColumnHandle(com.facebook.presto.spi.ColumnHandle) ScheduledSplit(com.facebook.presto.ScheduledSplit) Duration(io.airlift.units.Duration) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) FixedPageSource(com.facebook.presto.spi.FixedPageSource) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) PageConsumerOperator(com.facebook.presto.testing.PageConsumerOperator) Type(com.facebook.presto.spi.type.Type) ScheduledSplit(com.facebook.presto.ScheduledSplit) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) Split(com.facebook.presto.metadata.Split) TaskSource(com.facebook.presto.TaskSource) Session(com.facebook.presto.Session) Test(org.testng.annotations.Test)

Example 4 with ScheduledSplit

use of com.facebook.presto.ScheduledSplit in project presto by prestodb.

the class SqlTaskExecution method updateSources.

private synchronized Map<PlanNodeId, TaskSource> updateSources(List<TaskSource> sources) {
    Map<PlanNodeId, TaskSource> updatedUnpartitionedSources = new HashMap<>();
    // first remove any split that was already acknowledged
    long currentMaxAcknowledgedSplit = this.maxAcknowledgedSplit;
    sources = sources.stream().map(source -> new TaskSource(source.getPlanNodeId(), source.getSplits().stream().filter(scheduledSplit -> scheduledSplit.getSequenceId() > currentMaxAcknowledgedSplit).collect(Collectors.toSet()), source.isNoMoreSplits())).collect(toList());
    // update task with new sources
    for (TaskSource source : sources) {
        if (partitionedDriverFactories.containsKey(source.getPlanNodeId())) {
            schedulePartitionedSource(source);
        } else {
            scheduleUnpartitionedSource(source, updatedUnpartitionedSources);
        }
    }
    // update maxAcknowledgedSplit
    maxAcknowledgedSplit = sources.stream().flatMap(source -> source.getSplits().stream()).mapToLong(ScheduledSplit::getSequenceId).max().orElse(maxAcknowledgedSplit);
    return updatedUnpartitionedSources;
}
Also used : PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) LocalExecutionPlanner(com.facebook.presto.sql.planner.LocalExecutionPlanner) SystemSessionProperties.getSplitConcurrencyAdjustmentInterval(com.facebook.presto.SystemSessionProperties.getSplitConcurrencyAdjustmentInterval) OutputBuffer(com.facebook.presto.execution.buffer.OutputBuffer) TaskHandle(com.facebook.presto.execution.TaskExecutor.TaskHandle) Duration(io.airlift.units.Duration) SystemSessionProperties.getInitialSplitsPerNode(com.facebook.presto.SystemSessionProperties.getInitialSplitsPerNode) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) PlanFragment(com.facebook.presto.sql.planner.PlanFragment) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) QueryMonitor(com.facebook.presto.event.query.QueryMonitor) PipelineContext(com.facebook.presto.operator.PipelineContext) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) GuardedBy(javax.annotation.concurrent.GuardedBy) Collectors(java.util.stream.Collectors) ScheduledSplit(com.facebook.presto.ScheduledSplit) Preconditions.checkState(com.google.common.base.Preconditions.checkState) List(java.util.List) LocalExecutionPlan(com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) Entry(java.util.Map.Entry) Optional(java.util.Optional) SetThreadName(io.airlift.concurrent.SetThreadName) Queue(java.util.Queue) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) MoreObjects.toStringHelper(com.google.common.base.MoreObjects.toStringHelper) DriverStats(com.facebook.presto.operator.DriverStats) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) OptionalInt(java.util.OptionalInt) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) DriverFactory(com.facebook.presto.operator.DriverFactory) ImmutableList(com.google.common.collect.ImmutableList) TaskSource(com.facebook.presto.TaskSource) Objects.requireNonNull(java.util.Objects.requireNonNull) WeakReference(java.lang.ref.WeakReference) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) Nullable(javax.annotation.Nullable) TaskContext(com.facebook.presto.operator.TaskContext) Executor(java.util.concurrent.Executor) Throwables(com.google.common.base.Throwables) Driver(com.facebook.presto.operator.Driver) FutureCallback(com.google.common.util.concurrent.FutureCallback) Futures(com.google.common.util.concurrent.Futures) Collectors.toList(java.util.stream.Collectors.toList) StateChangeListener(com.facebook.presto.execution.StateMachine.StateChangeListener) BufferState(com.facebook.presto.execution.buffer.BufferState) DriverContext(com.facebook.presto.operator.DriverContext) ArrayDeque(java.util.ArrayDeque) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) TaskSource(com.facebook.presto.TaskSource)

Example 5 with ScheduledSplit

use of com.facebook.presto.ScheduledSplit in project presto by prestodb.

the class SqlTaskExecution method schedulePartitionedSource.

private synchronized void schedulePartitionedSource(TaskSource source) {
    // when the source is scheduled
    if (!isSchedulingSource(source.getPlanNodeId())) {
        pendingSplits.merge(source.getPlanNodeId(), source, TaskSource::update);
        return;
    }
    DriverSplitRunnerFactory partitionedDriverFactory = partitionedDriverFactories.get(source.getPlanNodeId());
    ImmutableList.Builder<DriverSplitRunner> runners = ImmutableList.builder();
    for (ScheduledSplit scheduledSplit : source.getSplits()) {
        // create a new driver for the split
        runners.add(partitionedDriverFactory.createDriverRunner(scheduledSplit, true));
    }
    enqueueDrivers(false, runners.build());
    if (source.isNoMoreSplits()) {
        partitionedDriverFactory.setNoMoreSplits();
        sourceStartOrder.remove(source.getPlanNodeId());
        // schedule next source
        if (!sourceStartOrder.isEmpty()) {
            TaskSource nextSource = pendingSplits.get(sourceStartOrder.peek());
            if (nextSource != null) {
                schedulePartitionedSource(nextSource);
            }
        }
    }
}
Also used : ScheduledSplit(com.facebook.presto.ScheduledSplit) ImmutableList(com.google.common.collect.ImmutableList) TaskSource(com.facebook.presto.TaskSource)

Aggregations

ScheduledSplit (com.facebook.presto.ScheduledSplit)5 TaskSource (com.facebook.presto.TaskSource)5 PlanNodeId (com.facebook.presto.sql.planner.plan.PlanNodeId)4 Split (com.facebook.presto.metadata.Split)3 Duration (io.airlift.units.Duration)3 Session (com.facebook.presto.Session)2 Driver (com.facebook.presto.operator.Driver)2 DriverContext (com.facebook.presto.operator.DriverContext)2 DriverFactory (com.facebook.presto.operator.DriverFactory)2 ColumnHandle (com.facebook.presto.spi.ColumnHandle)2 ConnectorPageSource (com.facebook.presto.spi.ConnectorPageSource)2 ConnectorSplit (com.facebook.presto.spi.ConnectorSplit)2 FixedPageSource (com.facebook.presto.spi.FixedPageSource)2 Type (com.facebook.presto.spi.type.Type)2 PageSourceProvider (com.facebook.presto.split.PageSourceProvider)2 LocalExecutionPlanner (com.facebook.presto.sql.planner.LocalExecutionPlanner)2 LocalExecutionPlan (com.facebook.presto.sql.planner.LocalExecutionPlanner.LocalExecutionPlan)2 ImmutableSet (com.google.common.collect.ImmutableSet)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2