use of io.prestosql.metadata.Split in project hetu-core by openlookeng.
the class TestNodeScheduler method testPrioritizedAssignmentOfLocalSplit.
@Test
public void testPrioritizedAssignmentOfLocalSplit() {
InternalNode node = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
nodeManager.addNode(CONNECTOR_ID, node);
// Check for Split assignments till maxSplitsPerNode (20)
Set<Split> splits = new LinkedHashSet<>();
// 20 splits with node1 as a non-local node to be assigned in the second iteration of computeAssignments
for (int i = 0; i < 20; i++) {
splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
}
// computeAssignments just returns a mapping of nodes with splits to be assigned, it does not assign splits
Multimap<InternalNode, Split> initialAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
// Check that all splits are being assigned to node1
assertEquals(initialAssignment.size(), 20);
assertEquals(initialAssignment.keySet().size(), 1);
assertTrue(initialAssignment.keySet().contains(node));
// Check for assignment of splits beyond maxSplitsPerNode (2 splits should remain unassigned)
// 1 split with node1 as local node
splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
// 1 split with node1 as a non-local node
splits.add(new Split(CONNECTOR_ID, new TestSplitRemote(), Lifespan.taskWide()));
// splits now contains 22 splits : 1 with node1 as local node and 21 with node1 as a non-local node
Multimap<InternalNode, Split> finalAssignment = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
// Check that only 20 splits are being assigned as there is a single task
assertEquals(finalAssignment.size(), 20);
assertEquals(finalAssignment.keySet().size(), 1);
assertTrue(finalAssignment.keySet().contains(node));
// When optimized-local-scheduling is enabled, the split with node1 as local node should be assigned
long countLocalSplits = finalAssignment.values().stream().map(Split::getConnectorSplit).filter(TestSplitLocal.class::isInstance).count();
assertEquals(countLocalSplits, 1);
}
use of io.prestosql.metadata.Split in project hetu-core by openlookeng.
the class TestNodeScheduler method testRuseExchangeComputeAssignmentsSplitsNotMatchProdConsumer.
@Test
public void testRuseExchangeComputeAssignmentsSplitsNotMatchProdConsumer() {
setUpNodes();
Split split = new Split(CONNECTOR_ID, new TestSplitLocallyAccessible(), Lifespan.taskWide());
Set<Split> splits = ImmutableSet.of(split);
NodeTaskMap newNodeTaskMap = new NodeTaskMap(new FinalizerService());
StageId stageId = new StageId(new QueryId("query"), 0);
UUID uuid = UUID.randomUUID();
PlanFragment testFragmentProducer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER, uuid, 1);
PlanNodeId tableScanNodeId = new PlanNodeId("plan_id");
StageExecutionPlan producerStageExecutionPlan = new StageExecutionPlan(testFragmentProducer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
SqlStageExecution producerStage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), producerStageExecutionPlan.getFragment(), producerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, newNodeTaskMap, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
nodeSelector.computeAssignments(splits, ImmutableList.copyOf(this.taskMap.values()), Optional.of(producerStage)).getAssignments().entries();
// Consumer
Split splitConsumer = new Split(CONNECTOR_ID, new TestSplitLocallyAccessibleDifferentIndex(), Lifespan.taskWide());
Set<Split> splitConsumers = ImmutableSet.of(splitConsumer);
PlanFragment testFragmentConsumer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER, uuid, 1);
StageExecutionPlan consumerStageExecutionPlan = new StageExecutionPlan(testFragmentConsumer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
SqlStageExecution stage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), consumerStageExecutionPlan.getFragment(), consumerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, newNodeTaskMap, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
try {
nodeSelector.computeAssignments(splitConsumers, ImmutableList.copyOf(this.taskMap.values()), Optional.of(stage)).getAssignments().entries();
} catch (PrestoException e) {
assertEquals("Producer & consumer splits are not same", e.getMessage());
return;
}
assertEquals(false, true);
}
use of io.prestosql.metadata.Split in project hetu-core by openlookeng.
the class TestNodeScheduler method testEmptyAssignmentWithFullNodes.
@Test
public void testEmptyAssignmentWithFullNodes() {
InternalNode node1 = new InternalNode("node1", URI.create("http://10.0.0.1:11"), NodeVersion.UNKNOWN, false);
nodeManager.addNode(CONNECTOR_ID, node1);
InternalNode node2 = new InternalNode("node2", URI.create("http://10.0.0.1:12"), NodeVersion.UNKNOWN, false);
nodeManager.addNode(CONNECTOR_ID, node2);
Set<Split> splits = new LinkedHashSet<>();
// 20 splits with node1 as local node to be assigned in the first iteration of computeAssignments
for (int i = 0; i < (20 + 10 + 5) * 2; i++) {
splits.add(new Split(CONNECTOR_ID, new TestSplitLocal(), Lifespan.taskWide()));
}
// computeAssignments just returns a mapping of nodes with splits to be assigned, it does not assign splits
Multimap<InternalNode, Split> assignments1 = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
assertEquals(assignments1.size(), 40);
assertEquals(assignments1.keySet().size(), 2);
assertEquals(assignments1.get(node1).size(), 20);
assertEquals(assignments1.get(node2).size(), 20);
MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
int task = 0;
for (InternalNode node : assignments1.keySet()) {
TaskId taskId = new TaskId("test", 1, task);
task++;
MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, ImmutableList.copyOf(assignments1.get(node)), nodeTaskMap.createPartitionedSplitCountTracker(node, taskId));
remoteTask.startSplits(20);
nodeTaskMap.addTask(node, remoteTask);
taskMap.put(node, remoteTask);
}
Set<Split> unassignedSplits = Sets.difference(splits, new HashSet<>(assignments1.values()));
assertEquals(unassignedSplits.size(), 30);
Multimap<InternalNode, Split> assignments2 = nodeSelector.computeAssignments(unassignedSplits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
for (InternalNode node : assignments2.keySet()) {
RemoteTask remoteTask = taskMap.get(node);
remoteTask.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), assignments2.get(node)).build());
}
unassignedSplits = Sets.difference(unassignedSplits, new HashSet<>(assignments2.values()));
assertEquals(unassignedSplits.size(), 10);
Multimap<InternalNode, Split> assignments3 = nodeSelector.computeAssignments(unassignedSplits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments();
assertTrue(assignments3.isEmpty());
}
use of io.prestosql.metadata.Split in project hetu-core by openlookeng.
the class TestNodeScheduler method testScheduleLocal.
@Test
public void testScheduleLocal() {
setUpNodes();
Split split = new Split(CONNECTOR_ID, new TestSplitLocallyAccessible(), Lifespan.taskWide());
Set<Split> splits = ImmutableSet.of(split);
Map.Entry<InternalNode, Split> assignment = Iterables.getOnlyElement(nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values()), Optional.empty()).getAssignments().entries());
assertEquals(assignment.getKey().getHostAndPort(), split.getAddresses().get(0));
assertEquals(assignment.getValue(), split);
}
use of io.prestosql.metadata.Split in project hetu-core by openlookeng.
the class TestIndexCacheFetch method setupBeforeClass.
@BeforeClass
public void setupBeforeClass() {
PropertyService.setProperty(HetuConstant.FILTER_ENABLED, true);
PropertyService.setProperty(HetuConstant.INDEXSTORE_FILESYSTEM_PROFILE, "local-config-default");
PropertyService.setProperty(HetuConstant.FILTER_CACHE_MAX_MEMORY, (long) (new DataSize(numberOfIndexTypes * 2, KILOBYTE).getValue(KILOBYTE)));
PropertyService.setProperty(HetuConstant.FILTER_CACHE_TTL, new Duration(10, TimeUnit.MINUTES));
PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_DELAY, new Duration(loadDelay, TimeUnit.MILLISECONDS));
PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_THREADS, 2L);
PropertyService.setProperty(HetuConstant.FILTER_CACHE_SOFT_REFERENCE, false);
CatalogName catalogName = new CatalogName(catalog);
connectorSplit = mock(ConnectorSplit.class);
Lifespan lifespan = mock(Lifespan.class);
split = new Split(catalogName, connectorSplit, lifespan);
when(connectorSplit.getFilePath()).thenReturn(testPath);
when(connectorSplit.getLastModifiedTime()).thenReturn(testLastModifiedTime);
}
Aggregations