Search in sources :

Example 6 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class TestSqlStage method createExchangePlanFragment.

private static PlanFragment createExchangePlanFragment() {
    PlanNode planNode = new RemoteSourceNode(new PlanNodeId("exchange"), ImmutableList.of(new PlanFragmentId("source")), ImmutableList.of(new Symbol("column")), Optional.empty(), REPARTITION, RetryPolicy.NONE);
    ImmutableMap.Builder<Symbol, Type> types = ImmutableMap.builder();
    for (Symbol symbol : planNode.getOutputSymbols()) {
        types.put(symbol, VARCHAR);
    }
    return new PlanFragment(new PlanFragmentId("exchange_fragment_id"), planNode, types.buildOrThrow(), SOURCE_DISTRIBUTION, ImmutableList.of(planNode.getId()), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), planNode.getOutputSymbols()), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty());
}
Also used : PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) Type(io.trino.spi.type.Type) PlanNode(io.trino.sql.planner.plan.PlanNode) Symbol(io.trino.sql.planner.Symbol) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) ImmutableMap(com.google.common.collect.ImmutableMap) PlanFragment(io.trino.sql.planner.PlanFragment)

Example 7 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class MockRemoteTaskFactory method createTableScanTask.

public MockRemoteTask createTableScanTask(TaskId taskId, InternalNode newNode, List<Split> splits, PartitionedSplitCountTracker partitionedSplitCountTracker) {
    Symbol symbol = new Symbol("column");
    PlanNodeId sourceId = new PlanNodeId("sourceId");
    PlanFragment testFragment = new PlanFragment(new PlanFragmentId("test"), TableScanNode.newInstance(sourceId, TEST_TABLE_HANDLE, ImmutableList.of(symbol), ImmutableMap.of(symbol, new TestingColumnHandle("column")), false, Optional.empty()), ImmutableMap.of(symbol, VARCHAR), SOURCE_DISTRIBUTION, ImmutableList.of(sourceId), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(symbol)), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty());
    ImmutableMultimap.Builder<PlanNodeId, Split> initialSplits = ImmutableMultimap.builder();
    for (Split sourceSplit : splits) {
        initialSplits.put(sourceId, sourceSplit);
    }
    return createRemoteTask(TEST_SESSION, taskId, newNode, testFragment, initialSplits.build(), createInitialEmptyOutputBuffers(BROADCAST), partitionedSplitCountTracker, ImmutableSet.of(), true);
}
Also used : PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) TestingColumnHandle(io.trino.testing.TestingMetadata.TestingColumnHandle) Symbol(io.trino.sql.planner.Symbol) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) Split(io.trino.metadata.Split) PlanFragment(io.trino.sql.planner.PlanFragment)

Example 8 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class GraphvizPrinter method printLogical.

public static String printLogical(List<PlanFragment> fragments) {
    Map<PlanFragmentId, PlanFragment> fragmentsById = Maps.uniqueIndex(fragments, PlanFragment::getId);
    PlanNodeIdGenerator idGenerator = new PlanNodeIdGenerator();
    StringBuilder output = new StringBuilder();
    output.append("digraph logical_plan {\n");
    for (PlanFragment fragment : fragments) {
        printFragmentNodes(output, fragment, idGenerator);
    }
    for (PlanFragment fragment : fragments) {
        fragment.getRoot().accept(new EdgePrinter(output, fragmentsById, idGenerator), null);
    }
    output.append("}\n");
    return output.toString();
}
Also used : PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) PlanFragment(io.trino.sql.planner.PlanFragment)

Example 9 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class TestSourcePartitionedScheduler method createStageExecution.

private StageExecution createStageExecution(PlanFragment fragment, NodeTaskMap nodeTaskMap) {
    StageId stageId = new StageId(QUERY_ID, 0);
    SqlStage stage = SqlStage.createSqlStage(stageId, fragment, ImmutableMap.of(TABLE_SCAN_NODE_ID, new TableInfo(new QualifiedObjectName("test", "test", "test"), TupleDomain.all())), new MockRemoteTaskFactory(queryExecutor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, queryExecutor, new SplitSchedulerStats());
    ImmutableMap.Builder<PlanFragmentId, OutputBufferManager> outputBuffers = ImmutableMap.builder();
    outputBuffers.put(fragment.getId(), new PartitionedOutputBufferManager(FIXED_HASH_DISTRIBUTION, 1));
    fragment.getRemoteSourceNodes().stream().flatMap(node -> node.getSourceFragmentIds().stream()).forEach(fragmentId -> outputBuffers.put(fragmentId, new PartitionedOutputBufferManager(FIXED_HASH_DISTRIBUTION, 10)));
    return createPipelinedStageExecution(stage, outputBuffers.buildOrThrow(), TaskLifecycleListener.NO_OP, new NoOpFailureDetector(), queryExecutor, Optional.of(new int[] { 0 }), 0);
}
Also used : PlanFragment(io.trino.sql.planner.PlanFragment) PartitionedSplitsInfo(io.trino.execution.PartitionedSplitsInfo) FIXED_HASH_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.FIXED_HASH_DISTRIBUTION) Test(org.testng.annotations.Test) TypeOperators(io.trino.spi.type.TypeOperators) TrinoExceptionAssert.assertTrinoExceptionThrownBy(io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy) FixedSplitSource(io.trino.spi.connector.FixedSplitSource) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) CatalogName(io.trino.connector.CatalogName) FinalizerService(io.trino.util.FinalizerService) ConnectorPartitionHandle(io.trino.spi.connector.ConnectorPartitionHandle) DynamicFilters(io.trino.sql.DynamicFilters) PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) TEST_SESSION(io.trino.SessionTestUtils.TEST_SESSION) Assert.assertFalse(org.testng.Assert.assertFalse) TableScanNode(io.trino.sql.planner.plan.TableScanNode) SplitsBalancingPolicy(io.trino.execution.scheduler.NodeSchedulerConfig.SplitsBalancingPolicy) NOT_PARTITIONED(io.trino.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) InMemoryNodeManager(io.trino.metadata.InMemoryNodeManager) NODE(io.trino.execution.scheduler.NodeSchedulerConfig.SplitsBalancingPolicy.NODE) TestingSession(io.trino.testing.TestingSession) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) Session(io.trino.Session) InternalNodeManager(io.trino.metadata.InternalNodeManager) StageId(io.trino.execution.StageId) ConnectorAwareSplitSource(io.trino.split.ConnectorAwareSplitSource) Supplier(java.util.function.Supplier) REPLICATE(io.trino.sql.planner.plan.ExchangeNode.Type.REPLICATE) StatsAndCosts(io.trino.cost.StatsAndCosts) DynamicFilterId(io.trino.sql.planner.plan.DynamicFilterId) ArrayList(java.util.ArrayList) STAGE(io.trino.execution.scheduler.NodeSchedulerConfig.SplitsBalancingPolicy.STAGE) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) TestingSplit(io.trino.testing.TestingSplit) StageExecutionDescriptor.ungroupedExecution(io.trino.operator.StageExecutionDescriptor.ungroupedExecution) NodeVersion(io.trino.client.NodeVersion) MockRemoteTaskFactory(io.trino.execution.MockRemoteTaskFactory) NoOpFailureDetector(io.trino.failuredetector.NoOpFailureDetector) Integer.min(java.lang.Integer.min) AfterClass(org.testng.annotations.AfterClass) SqlStage(io.trino.execution.SqlStage) ConnectorSplit(io.trino.spi.connector.ConnectorSplit) ConnectorSplitSource(io.trino.spi.connector.ConnectorSplitSource) MockRemoteTask(io.trino.execution.MockRemoteTaskFactory.MockRemoteTask) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) QueryId(io.trino.spi.QueryId) NodeTaskMap(io.trino.execution.NodeTaskMap) SymbolAllocator(io.trino.sql.planner.SymbolAllocator) SCHEDULING(io.trino.execution.scheduler.StageExecution.State.SCHEDULING) FilterNode(io.trino.sql.planner.plan.FilterNode) Duration(io.airlift.units.Duration) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) URI(java.net.URI) SINGLE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) JoinNode(io.trino.sql.planner.plan.JoinNode) DynamicFilterService(io.trino.server.DynamicFilterService) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(io.trino.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) SPLIT_QUEUES_FULL(io.trino.execution.scheduler.ScheduleResult.BlockedReason.SPLIT_QUEUES_FULL) BeforeClass(org.testng.annotations.BeforeClass) DynamicFilters.createDynamicFilterExpression(io.trino.sql.DynamicFilters.createDynamicFilterExpression) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) TableInfo(io.trino.execution.TableInfo) PLANNED(io.trino.execution.scheduler.StageExecution.State.PLANNED) DynamicFilter(io.trino.spi.connector.DynamicFilter) MetadataManager.createTestMetadataManager(io.trino.metadata.MetadataManager.createTestMetadataManager) Optional(java.util.Optional) NO_NODES_AVAILABLE(io.trino.spi.StandardErrorCode.NO_NODES_AVAILABLE) TestingColumnHandle(io.trino.testing.TestingMetadata.TestingColumnHandle) INNER(io.trino.sql.planner.plan.JoinNode.Type.INNER) Assert.assertEquals(org.testng.Assert.assertEquals) CompletableFuture(java.util.concurrent.CompletableFuture) RetryPolicy(io.trino.operator.RetryPolicy) Partitioning(io.trino.sql.planner.Partitioning) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) FunctionManager.createTestingFunctionManager(io.trino.metadata.FunctionManager.createTestingFunctionManager) ImmutableList(com.google.common.collect.ImmutableList) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) Objects.requireNonNull(java.util.Objects.requireNonNull) DynamicFilterConfig(io.trino.execution.DynamicFilterConfig) ExecutorService(java.util.concurrent.ExecutorService) Symbol(io.trino.sql.planner.Symbol) TableExecuteContextManager(io.trino.execution.TableExecuteContextManager) RemoteTask(io.trino.execution.RemoteTask) TupleDomain(io.trino.spi.predicate.TupleDomain) FunctionManager(io.trino.metadata.FunctionManager) InternalNode(io.trino.metadata.InternalNode) TEST_TABLE_HANDLE(io.trino.testing.TestingHandles.TEST_TABLE_HANDLE) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) PipelinedStageExecution.createPipelinedStageExecution(io.trino.execution.scheduler.PipelinedStageExecution.createPipelinedStageExecution) SOURCE_DISTRIBUTION(io.trino.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) Metadata(io.trino.metadata.Metadata) Assert.assertTrue(org.testng.Assert.assertTrue) SECONDS(java.util.concurrent.TimeUnit.SECONDS) NoOpFailureDetector(io.trino.failuredetector.NoOpFailureDetector) SqlStage(io.trino.execution.SqlStage) StageId(io.trino.execution.StageId) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) ImmutableMap(com.google.common.collect.ImmutableMap) TableInfo(io.trino.execution.TableInfo) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) MockRemoteTaskFactory(io.trino.execution.MockRemoteTaskFactory)

Example 10 with PlanFragmentId

use of io.trino.sql.planner.plan.PlanFragmentId in project trino by trinodb.

the class TestSourcePartitionedScheduler method createFragment.

private static PlanFragment createFragment() {
    Symbol symbol = new Symbol("column");
    Symbol buildSymbol = new Symbol("buildColumn");
    // table scan with splitCount splits
    TableScanNode tableScan = TableScanNode.newInstance(TABLE_SCAN_NODE_ID, TEST_TABLE_HANDLE, ImmutableList.of(symbol), ImmutableMap.of(symbol, new TestingColumnHandle("column")), false, Optional.empty());
    FilterNode filterNode = new FilterNode(new PlanNodeId("filter_node_id"), tableScan, createDynamicFilterExpression(TEST_SESSION, createTestMetadataManager(), DYNAMIC_FILTER_ID, VARCHAR, symbol.toSymbolReference()));
    RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("remote_id"), new PlanFragmentId("plan_fragment_id"), ImmutableList.of(buildSymbol), Optional.empty(), REPLICATE, RetryPolicy.NONE);
    return new PlanFragment(new PlanFragmentId("plan_id"), new JoinNode(new PlanNodeId("join_id"), INNER, filterNode, remote, ImmutableList.of(), tableScan.getOutputSymbols(), remote.getOutputSymbols(), false, Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of(DYNAMIC_FILTER_ID, buildSymbol), Optional.empty()), ImmutableMap.of(symbol, VARCHAR), SOURCE_DISTRIBUTION, ImmutableList.of(TABLE_SCAN_NODE_ID), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(symbol)), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty());
}
Also used : PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) TestingColumnHandle(io.trino.testing.TestingMetadata.TestingColumnHandle) RemoteSourceNode(io.trino.sql.planner.plan.RemoteSourceNode) TableScanNode(io.trino.sql.planner.plan.TableScanNode) Symbol(io.trino.sql.planner.Symbol) JoinNode(io.trino.sql.planner.plan.JoinNode) PartitioningScheme(io.trino.sql.planner.PartitioningScheme) FilterNode(io.trino.sql.planner.plan.FilterNode) PlanFragmentId(io.trino.sql.planner.plan.PlanFragmentId) PlanFragment(io.trino.sql.planner.PlanFragment)

Aggregations

PlanFragmentId (io.trino.sql.planner.plan.PlanFragmentId)22 PlanFragment (io.trino.sql.planner.PlanFragment)15 PlanNodeId (io.trino.sql.planner.plan.PlanNodeId)10 RemoteSourceNode (io.trino.sql.planner.plan.RemoteSourceNode)10 Test (org.testng.annotations.Test)7 PartitioningScheme (io.trino.sql.planner.PartitioningScheme)6 Symbol (io.trino.sql.planner.Symbol)6 ImmutableMap (com.google.common.collect.ImmutableMap)5 ImmutableSet (com.google.common.collect.ImmutableSet)5 FragmentsEdge (io.trino.execution.scheduler.policy.PhasedExecutionSchedule.FragmentsEdge)5 PlanUtils.createBroadcastAndPartitionedJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createBroadcastAndPartitionedJoinPlanFragment)5 PlanUtils.createBroadcastJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createBroadcastJoinPlanFragment)5 PlanUtils.createJoinPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createJoinPlanFragment)5 PlanUtils.createTableScanPlanFragment (io.trino.execution.scheduler.policy.PlanUtils.createTableScanPlanFragment)5 JoinNode (io.trino.sql.planner.plan.JoinNode)5 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)4 ExchangeSourceHandle (io.trino.spi.exchange.ExchangeSourceHandle)4 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)3