Search in sources :

Example 1 with LocalExecutionPlanner

use of io.trino.sql.planner.LocalExecutionPlanner in project trino by trinodb.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    PageSourceManager pageSourceManager = new PageSourceManager();
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    BlockTypeOperators blockTypeOperators = new BlockTypeOperators(PLANNER_CONTEXT.getTypeOperators());
    NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService)));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, blockTypeOperators);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(PLANNER_CONTEXT.getFunctionManager(), 0);
    return new LocalExecutionPlanner(PLANNER_CONTEXT, createTestingTypeAnalyzer(PLANNER_CONTEXT), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockDirectExchangeClientSupplier(), new ExpressionCompiler(PLANNER_CONTEXT.getFunctionManager(), pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(PLANNER_CONTEXT.getFunctionManager()), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(PLANNER_CONTEXT.getTypeOperators()), new TrinoOperatorFactories(), new OrderingCompiler(PLANNER_CONTEXT.getTypeOperators()), new DynamicFilterConfig(), blockTypeOperators, new TableExecuteContextManager(), new ExchangeManagerRegistry(new ExchangeHandleResolver()));
}
Also used : PlanFragment(io.trino.sql.planner.PlanFragment) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) ObjectMapperProvider(io.airlift.json.ObjectMapperProvider) CatalogName(io.trino.connector.CatalogName) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) FinalizerService(io.trino.util.FinalizerService) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) TEST_SESSION(io.trino.SessionTestUtils.TEST_SESSION) SINGLE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) TableScanNode(io.trino.sql.planner.plan.TableScanNode) ImmutableMap(com.google.common.collect.ImmutableMap) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PageSinkManager(io.trino.split.PageSinkManager) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) SplitMonitor(io.trino.event.SplitMonitor) MockDirectExchangeClientSupplier(io.trino.execution.TestSqlTaskManager.MockDirectExchangeClientSupplier) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) Split(io.trino.metadata.Split) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) Optional(java.util.Optional) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) TestingColumnHandle(io.trino.testing.TestingMetadata.TestingColumnHandle) TypeAnalyzer.createTestingTypeAnalyzer(io.trino.sql.planner.TypeAnalyzer.createTestingTypeAnalyzer) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) Partitioning(io.trino.sql.planner.Partitioning) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) StatsAndCosts(io.trino.cost.StatsAndCosts) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TestingSplit(io.trino.testing.TestingSplit) StageExecutionDescriptor.ungroupedExecution(io.trino.operator.StageExecutionDescriptor.ungroupedExecution) EventListenerConfig(io.trino.eventlistener.EventListenerConfig) IndexManager(io.trino.index.IndexManager) PagesIndex(io.trino.operator.PagesIndex) Symbol(io.trino.sql.planner.Symbol) BlockTypeOperators(io.trino.type.BlockTypeOperators) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSourceManager(io.trino.split.PageSourceManager) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) PLANNER_CONTEXT(io.trino.sql.planner.TestingPlannerContext.PLANNER_CONTEXT) EventListenerManager(io.trino.eventlistener.EventListenerManager) TEST_TABLE_HANDLE(io.trino.testing.TestingHandles.TEST_TABLE_HANDLE) OutputBuffers(io.trino.execution.buffer.OutputBuffers) SOURCE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) PageFunctionCompiler(io.trino.sql.gen.PageFunctionCompiler) MockDirectExchangeClientSupplier(io.trino.execution.TestSqlTaskManager.MockDirectExchangeClientSupplier) TrinoOperatorFactories(io.trino.operator.TrinoOperatorFactories) NodeSchedulerConfig(io.trino.execution.scheduler.NodeSchedulerConfig) PagesIndex(io.trino.operator.PagesIndex) PageSourceManager(io.trino.split.PageSourceManager) NodePartitioningManager(io.trino.sql.planner.NodePartitioningManager) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) BlockTypeOperators(io.trino.type.BlockTypeOperators) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) GenericSpillerFactory(io.trino.spiller.GenericSpillerFactory) PageSinkManager(io.trino.split.PageSinkManager) JoinCompiler(io.trino.sql.gen.JoinCompiler) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) UniformNodeSelectorFactory(io.trino.execution.scheduler.UniformNodeSelectorFactory) JoinFilterFunctionCompiler(io.trino.sql.gen.JoinFilterFunctionCompiler) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) IndexManager(io.trino.index.IndexManager) FinalizerService(io.trino.util.FinalizerService) ExpressionCompiler(io.trino.sql.gen.ExpressionCompiler)

Example 2 with LocalExecutionPlanner

use of io.trino.sql.planner.LocalExecutionPlanner in project trino by trinodb.

the class TestMemoryRevokingScheduler method setUp.

@BeforeMethod
public void setUp() {
    memoryPool = new MemoryPool(DataSize.ofBytes(10));
    TaskExecutor taskExecutor = new TaskExecutor(8, 16, 3, 4, Ticker.systemTicker());
    taskExecutor.start();
    // Must be single threaded
    executor = newScheduledThreadPool(1, threadsNamed("task-notification-%s"));
    scheduledExecutor = newScheduledThreadPool(2, threadsNamed("task-notification-%s"));
    LocalExecutionPlanner planner = createTestingPlanner();
    sqlTaskExecutionFactory = new SqlTaskExecutionFactory(executor, taskExecutor, planner, createTestSplitMonitor(), new TaskManagerConfig());
    allOperatorContexts = null;
}
Also used : TaskExecutor(io.trino.execution.executor.TaskExecutor) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) MemoryPool(io.trino.memory.MemoryPool) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 3 with LocalExecutionPlanner

use of io.trino.sql.planner.LocalExecutionPlanner in project trino by trinodb.

the class LocalQueryRunner method createDrivers.

private List<Driver> createDrivers(Session session, Plan plan, OutputFactory outputFactory, TaskContext taskContext) {
    if (printPlan) {
        System.out.println(PlanPrinter.textLogicalPlan(plan.getRoot(), plan.getTypes(), plannerContext.getMetadata(), plannerContext.getFunctionManager(), plan.getStatsAndCosts(), session, 0, false));
    }
    SubPlan subplan = createSubPlans(session, plan, true);
    if (!subplan.getChildren().isEmpty()) {
        throw new AssertionError("Expected subplan to have no children");
    }
    TableExecuteContextManager tableExecuteContextManager = new TableExecuteContextManager();
    tableExecuteContextManager.registerTableExecuteContextForQuery(taskContext.getQueryContext().getQueryId());
    LocalExecutionPlanner executionPlanner = new LocalExecutionPlanner(plannerContext, new TypeAnalyzer(plannerContext, statementAnalyzerFactory), Optional.empty(), pageSourceManager, indexManager, nodePartitioningManager, pageSinkManager, null, expressionCompiler, pageFunctionCompiler, joinFilterFunctionCompiler, new IndexJoinLookupStats(), this.taskManagerConfig, spillerFactory, singleStreamSpillerFactory, partitioningSpillerFactory, new PagesIndex.TestingFactory(false), joinCompiler, operatorFactories, new OrderingCompiler(plannerContext.getTypeOperators()), new DynamicFilterConfig(), blockTypeOperators, tableExecuteContextManager, exchangeManagerRegistry);
    // plan query
    StageExecutionDescriptor stageExecutionDescriptor = subplan.getFragment().getStageExecutionDescriptor();
    LocalExecutionPlan localExecutionPlan = executionPlanner.plan(taskContext, stageExecutionDescriptor, subplan.getFragment().getRoot(), subplan.getFragment().getPartitioningScheme().getOutputLayout(), plan.getTypes(), subplan.getFragment().getPartitionedSources(), outputFactory);
    // generate splitAssignments
    List<SplitAssignment> splitAssignments = new ArrayList<>();
    long sequenceId = 0;
    for (TableScanNode tableScan : findTableScanNodes(subplan.getFragment().getRoot())) {
        TableHandle table = tableScan.getTable();
        SplitSource splitSource = splitManager.getSplits(session, table, stageExecutionDescriptor.isScanGroupedExecution(tableScan.getId()) ? GROUPED_SCHEDULING : UNGROUPED_SCHEDULING, EMPTY, alwaysTrue());
        ImmutableSet.Builder<ScheduledSplit> scheduledSplits = ImmutableSet.builder();
        while (!splitSource.isFinished()) {
            for (Split split : getNextBatch(splitSource)) {
                scheduledSplits.add(new ScheduledSplit(sequenceId++, tableScan.getId(), split));
            }
        }
        splitAssignments.add(new SplitAssignment(tableScan.getId(), scheduledSplits.build(), true));
    }
    // create drivers
    List<Driver> drivers = new ArrayList<>();
    Map<PlanNodeId, DriverFactory> driverFactoriesBySource = new HashMap<>();
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        for (int i = 0; i < driverFactory.getDriverInstances().orElse(1); i++) {
            if (driverFactory.getSourceId().isPresent()) {
                checkState(driverFactoriesBySource.put(driverFactory.getSourceId().get(), driverFactory) == null);
            } else {
                DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver(), false).addDriverContext();
                Driver driver = driverFactory.createDriver(driverContext);
                drivers.add(driver);
            }
        }
    }
    // add split assignments to the drivers
    ImmutableSet<PlanNodeId> partitionedSources = ImmutableSet.copyOf(subplan.getFragment().getPartitionedSources());
    for (SplitAssignment splitAssignment : splitAssignments) {
        DriverFactory driverFactory = driverFactoriesBySource.get(splitAssignment.getPlanNodeId());
        checkState(driverFactory != null);
        boolean partitioned = partitionedSources.contains(driverFactory.getSourceId().get());
        for (ScheduledSplit split : splitAssignment.getSplits()) {
            DriverContext driverContext = taskContext.addPipelineContext(driverFactory.getPipelineId(), driverFactory.isInputDriver(), driverFactory.isOutputDriver(), partitioned).addDriverContext();
            Driver driver = driverFactory.createDriver(driverContext);
            driver.updateSplitAssignment(new SplitAssignment(split.getPlanNodeId(), ImmutableSet.of(split), true));
            drivers.add(driver);
        }
    }
    for (DriverFactory driverFactory : localExecutionPlan.getDriverFactories()) {
        driverFactory.noMoreDrivers();
    }
    return ImmutableList.copyOf(drivers);
}
Also used : DriverContext(io.trino.operator.DriverContext) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Driver(io.trino.operator.Driver) SplitAssignment(io.trino.execution.SplitAssignment) PagesIndex(io.trino.operator.PagesIndex) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) ImmutableSet(com.google.common.collect.ImmutableSet) TableExecuteContextManager(io.trino.execution.TableExecuteContextManager) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) DriverFactory(io.trino.operator.DriverFactory) ScheduledSplit(io.trino.execution.ScheduledSplit) LocalExecutionPlanner(io.trino.sql.planner.LocalExecutionPlanner) IndexJoinLookupStats(io.trino.operator.index.IndexJoinLookupStats) StageExecutionDescriptor(io.trino.operator.StageExecutionDescriptor) TypeAnalyzer(io.trino.sql.planner.TypeAnalyzer) LocalExecutionPlan(io.trino.sql.planner.LocalExecutionPlanner.LocalExecutionPlan) TableScanNode(io.trino.sql.planner.plan.TableScanNode) TableHandle(io.trino.metadata.TableHandle) SplitSource(io.trino.split.SplitSource) Split(io.trino.metadata.Split) ScheduledSplit(io.trino.execution.ScheduledSplit) SubPlan(io.trino.sql.planner.SubPlan) DynamicFilterConfig(io.trino.execution.DynamicFilterConfig)

Aggregations

LocalExecutionPlanner (io.trino.sql.planner.LocalExecutionPlanner)3 Split (io.trino.metadata.Split)2 PagesIndex (io.trino.operator.PagesIndex)2 IndexJoinLookupStats (io.trino.operator.index.IndexJoinLookupStats)2 OrderingCompiler (io.trino.sql.gen.OrderingCompiler)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 ObjectMapperProvider (io.airlift.json.ObjectMapperProvider)1 TEST_SESSION (io.trino.SessionTestUtils.TEST_SESSION)1 CatalogName (io.trino.connector.CatalogName)1 StatsAndCosts (io.trino.cost.StatsAndCosts)1 SplitMonitor (io.trino.event.SplitMonitor)1 EventListenerConfig (io.trino.eventlistener.EventListenerConfig)1 EventListenerManager (io.trino.eventlistener.EventListenerManager)1 ExchangeManagerRegistry (io.trino.exchange.ExchangeManagerRegistry)1 DynamicFilterConfig (io.trino.execution.DynamicFilterConfig)1 ScheduledSplit (io.trino.execution.ScheduledSplit)1 SplitAssignment (io.trino.execution.SplitAssignment)1 TableExecuteContextManager (io.trino.execution.TableExecuteContextManager)1