use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class TestPhasedExecutionSchedule method testUnion.
@Test
public void testUnion() {
PlanFragment aFragment = createTableScanPlanFragment("a");
PlanFragment bFragment = createTableScanPlanFragment("b");
PlanFragment cFragment = createTableScanPlanFragment("c");
PlanFragment unionFragment = createUnionPlanFragment("union", aFragment, bFragment, cFragment);
List<Set<PlanFragmentId>> phases = PhasedExecutionSchedule.extractPhases(ImmutableList.of(aFragment, bFragment, cFragment, unionFragment));
assertEquals(phases, ImmutableList.of(ImmutableSet.of(unionFragment.getId()), ImmutableSet.of(aFragment.getId()), ImmutableSet.of(bFragment.getId()), ImmutableSet.of(cFragment.getId())));
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class TestPhasedExecutionSchedule method testJoinWithDeepSources.
@Test
public void testJoinWithDeepSources() {
PlanFragment buildSourceFragment = createTableScanPlanFragment("buildSource");
PlanFragment buildMiddleFragment = createExchangePlanFragment("buildMiddle", buildSourceFragment);
PlanFragment buildTopFragment = createExchangePlanFragment("buildTop", buildMiddleFragment);
PlanFragment probeSourceFragment = createTableScanPlanFragment("probeSource");
PlanFragment probeMiddleFragment = createExchangePlanFragment("probeMiddle", probeSourceFragment);
PlanFragment probeTopFragment = createExchangePlanFragment("probeTop", probeMiddleFragment);
PlanFragment joinFragment = createJoinPlanFragment(INNER, "join", buildTopFragment, probeTopFragment);
List<Set<PlanFragmentId>> phases = PhasedExecutionSchedule.extractPhases(ImmutableList.of(joinFragment, buildTopFragment, buildMiddleFragment, buildSourceFragment, probeTopFragment, probeMiddleFragment, probeSourceFragment));
assertEquals(phases, ImmutableList.of(ImmutableSet.of(joinFragment.getId()), ImmutableSet.of(buildTopFragment.getId()), ImmutableSet.of(buildMiddleFragment.getId()), ImmutableSet.of(buildSourceFragment.getId()), ImmutableSet.of(probeTopFragment.getId()), ImmutableSet.of(probeMiddleFragment.getId()), ImmutableSet.of(probeSourceFragment.getId())));
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class TestSourcePartitionedScheduler method createPlan.
private static StageExecutionPlan createPlan(ConnectorSplitSource splitSource) {
Symbol symbol = new Symbol("column");
// table scan with splitCount splits
PlanNodeId tableScanNodeId = new PlanNodeId("plan_id");
TableScanNode tableScan = TableScanNode.newInstance(tableScanNodeId, TEST_TABLE_HANDLE, ImmutableList.of(symbol), ImmutableMap.of(symbol, new TestingColumnHandle("column")), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), 0, false);
RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("remote_id"), new PlanFragmentId("plan_fragment_id"), ImmutableList.of(), Optional.empty(), GATHER);
PlanFragment testFragment = new PlanFragment(new PlanFragmentId("plan_id"), new JoinNode(new PlanNodeId("join_id"), INNER, tableScan, remote, ImmutableList.of(), ImmutableList.<Symbol>builder().addAll(tableScan.getOutputSymbols()).addAll(remote.getOutputSymbols()).build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty(), ImmutableMap.of()), ImmutableMap.of(symbol, VARCHAR), SOURCE_DISTRIBUTION, ImmutableList.of(tableScanNodeId), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(symbol)), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty(), Optional.empty(), Optional.empty());
return new StageExecutionPlan(testFragment, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, splitSource)), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", "test", "test"), TupleDomain.all())));
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class TestHiveIntegrationSmokeTest method testRuseExchangeGroupSplitsMatchingBetweenProducerConsumer.
@Test
public void testRuseExchangeGroupSplitsMatchingBetweenProducerConsumer() {
setUpNodes();
NodeTaskMap nodeTasks = new NodeTaskMap(new FinalizerService());
StageId stageId = new StageId(new QueryId("query"), 0);
UUID uuid = UUID.randomUUID();
PlanFragment testFragmentProducer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER, uuid, 1);
PlanNodeId tableScanNodeId = new PlanNodeId("plan_id");
StageExecutionPlan producerStageExecutionPlan = new StageExecutionPlan(testFragmentProducer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
SqlStageExecution producerStage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), producerStageExecutionPlan.getFragment(), producerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, nodeTasks, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
Set<Split> splits = createAndGetSplits(10);
Multimap<InternalNode, Split> producerAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.of(producerStage)).getAssignments();
PlanFragment testFragmentConsumer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER, uuid, 1);
StageExecutionPlan consumerStageExecutionPlan = new StageExecutionPlan(testFragmentConsumer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
SqlStageExecution stage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), consumerStageExecutionPlan.getFragment(), consumerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, nodeTasks, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
Multimap<InternalNode, Split> consumerAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.of(stage)).getAssignments();
assertEquals(consumerAssignment.size(), consumerAssignment.size());
for (InternalNode node : consumerAssignment.keySet()) {
List<Split> splitList = new ArrayList<>();
List<Split> splitList2 = new ArrayList<>();
boolean b = producerAssignment.containsEntry(node, consumerAssignment.get(node));
Collection<Split> producerSplits = producerAssignment.get(node);
Collection<Split> consumerSplits = producerAssignment.get(node);
producerSplits.forEach(s -> splitList.add(s));
List<Split> splitList1 = splitList.get(0).getSplits();
consumerSplits.forEach(s -> splitList2.add(s));
int i = 0;
for (Split split3 : splitList1) {
SplitKey splitKey1 = new SplitKey(split3, TEST_CATALOG, TEST_SCHEMA, TEST_TABLE);
SplitKey splitKey2 = new SplitKey(splitList1.get(i), TEST_CATALOG, TEST_SCHEMA, TEST_TABLE);
boolean f = splitKey1.equals(splitKey2);
assertEquals(true, f);
i++;
}
}
}
use of io.prestosql.sql.planner.PlanFragment in project hetu-core by openlookeng.
the class TestStageStateMachine method createValuesPlan.
private static PlanFragment createValuesPlan() {
Symbol symbol = new Symbol("column");
PlanNodeId valuesNodeId = new PlanNodeId("plan");
PlanFragment planFragment = new PlanFragment(new PlanFragmentId("plan"), new ValuesNode(valuesNodeId, ImmutableList.of(symbol), ImmutableList.of(ImmutableList.of(castToRowExpression(new StringLiteral("foo"))))), ImmutableMap.of(symbol, VARCHAR), SOURCE_DISTRIBUTION, ImmutableList.of(valuesNodeId), new PartitioningScheme(Partitioning.create(SINGLE_DISTRIBUTION, ImmutableList.of()), ImmutableList.of(symbol)), ungroupedExecution(), StatsAndCosts.empty(), Optional.empty(), Optional.empty(), Optional.empty());
return planFragment;
}
Aggregations