use of io.trino.testing.TestingSplit in project trino by trinodb.
the class TestSourcePartitionedScheduler method testStageBalancedSplitAssignment.
@Test
public void testStageBalancedSplitAssignment() {
// use private node manager so we can add a node later
InMemoryNodeManager nodeManager = new InMemoryNodeManager();
nodeManager.addNode(CONNECTOR_ID, new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false));
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
// Schedule 15 splits - there are 3 nodes, each node should get 5 splits
PlanFragment firstPlan = createFragment();
StageExecution firstStage = createStageExecution(firstPlan, nodeTaskMap);
QueuedSplitSource firstSplitSource = new QueuedSplitSource(TestingSplit::createRemoteSplit);
StageScheduler firstScheduler = getSourcePartitionedScheduler(firstSplitSource, firstStage, nodeManager, nodeTaskMap, 200, STAGE);
firstSplitSource.addSplits(15);
ScheduleResult scheduleResult = firstScheduler.schedule();
assertTrue(scheduleResult.getBlocked().isDone());
assertEquals(scheduleResult.getNewTasks().size(), 3);
assertEquals(firstStage.getAllTasks().size(), 3);
for (RemoteTask remoteTask : firstStage.getAllTasks()) {
PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
assertEquals(splitsInfo.getCount(), 5);
}
// Add new node
InternalNode additionalNode = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
nodeManager.addNode(CONNECTOR_ID, additionalNode);
// Schedule 5 splits in first query. Since the new node does not have any splits, all 5 splits are assigned to the new node
firstSplitSource.addSplits(5);
firstSplitSource.close();
scheduleResult = firstScheduler.schedule();
assertEffectivelyFinished(scheduleResult, firstScheduler);
assertTrue(scheduleResult.getBlocked().isDone());
assertEquals(scheduleResult.getNewTasks().size(), 1);
assertEquals(firstStage.getAllTasks().size(), 4);
for (RemoteTask remoteTask : firstStage.getAllTasks()) {
PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
assertEquals(splitsInfo.getCount(), 5);
}
// Add new node
InternalNode anotherAdditionalNode = new InternalNode("other5", URI.create("http://127.0.0.1:15"), NodeVersion.UNKNOWN, false);
nodeManager.addNode(CONNECTOR_ID, anotherAdditionalNode);
// Schedule 5 splits in another query. New query should be balanced across all nodes
PlanFragment secondPlan = createFragment();
StageExecution secondStage = createStageExecution(secondPlan, nodeTaskMap);
StageScheduler secondScheduler = getSourcePartitionedScheduler(createFixedSplitSource(5, TestingSplit::createRemoteSplit), secondStage, nodeManager, nodeTaskMap, 200, STAGE);
scheduleResult = secondScheduler.schedule();
assertEffectivelyFinished(scheduleResult, secondScheduler);
assertEquals(secondStage.getAllTasks().size(), 5);
for (RemoteTask remoteTask : secondStage.getAllTasks()) {
PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
assertEquals(splitsInfo.getCount(), 1);
}
firstStage.abort();
secondStage.abort();
}
use of io.trino.testing.TestingSplit in project trino by trinodb.
the class TestSourcePartitionedScheduler method testNewTaskScheduledWhenChildStageBufferIsUnderutilized.
@Test
public void testNewTaskScheduledWhenChildStageBufferIsUnderutilized() {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
// use private node manager so we can add a node later
InMemoryNodeManager nodeManager = new InMemoryNodeManager();
nodeManager.addNode(CONNECTOR_ID, new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false));
NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(nodeManager, new NodeSchedulerConfig().setIncludeCoordinator(false), nodeTaskMap, new Duration(0, SECONDS)));
PlanFragment plan = createFragment();
StageExecution stage = createStageExecution(plan, nodeTaskMap);
// setting under utilized child output buffer
StageScheduler scheduler = newSourcePartitionedSchedulerAsStageScheduler(stage, TABLE_SCAN_NODE_ID, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(500, TestingSplit::createRemoteSplit)), new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(session, Optional.of(CONNECTOR_ID)), stage::getAllTasks), 500, new DynamicFilterService(metadata, functionManager, typeOperators, new DynamicFilterConfig()), new TableExecuteContextManager(), () -> false);
// the queues of 3 running nodes should be full
ScheduleResult scheduleResult = scheduler.schedule();
assertEquals(scheduleResult.getBlockedReason().get(), SPLIT_QUEUES_FULL);
assertEquals(scheduleResult.getNewTasks().size(), 3);
assertEquals(scheduleResult.getSplitsScheduled(), 300);
for (RemoteTask remoteTask : scheduleResult.getNewTasks()) {
PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
assertEquals(splitsInfo.getCount(), 100);
}
// new node added - the pending splits should go to it since the child tasks are not blocked
nodeManager.addNode(CONNECTOR_ID, new InternalNode("other4", URI.create("http://127.0.0.4:14"), NodeVersion.UNKNOWN, false));
scheduleResult = scheduler.schedule();
// split queue is full but still the source task creation isn't blocked
assertEquals(scheduleResult.getBlockedReason().get(), SPLIT_QUEUES_FULL);
assertEquals(scheduleResult.getNewTasks().size(), 1);
assertEquals(scheduleResult.getSplitsScheduled(), 100);
}
use of io.trino.testing.TestingSplit in project trino by trinodb.
the class TestSourcePartitionedScheduler method testNoNewTaskScheduledWhenChildStageBufferIsOverutilized.
@Test
public void testNoNewTaskScheduledWhenChildStageBufferIsOverutilized() {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
// use private node manager so we can add a node later
InMemoryNodeManager nodeManager = new InMemoryNodeManager();
nodeManager.addNode(CONNECTOR_ID, new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false), new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false), new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false));
NodeScheduler nodeScheduler = new NodeScheduler(new UniformNodeSelectorFactory(nodeManager, new NodeSchedulerConfig().setIncludeCoordinator(false), nodeTaskMap, new Duration(0, SECONDS)));
PlanFragment plan = createFragment();
StageExecution stage = createStageExecution(plan, nodeTaskMap);
// setting over utilized child output buffer
StageScheduler scheduler = newSourcePartitionedSchedulerAsStageScheduler(stage, TABLE_SCAN_NODE_ID, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(400, TestingSplit::createRemoteSplit)), new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(session, Optional.of(CONNECTOR_ID)), stage::getAllTasks), 400, new DynamicFilterService(metadata, functionManager, typeOperators, new DynamicFilterConfig()), new TableExecuteContextManager(), () -> true);
// the queues of 3 running nodes should be full
ScheduleResult scheduleResult = scheduler.schedule();
assertEquals(scheduleResult.getBlockedReason().get(), SPLIT_QUEUES_FULL);
assertEquals(scheduleResult.getNewTasks().size(), 3);
assertEquals(scheduleResult.getSplitsScheduled(), 300);
for (RemoteTask remoteTask : scheduleResult.getNewTasks()) {
PartitionedSplitsInfo splitsInfo = remoteTask.getPartitionedSplitsInfo();
assertEquals(splitsInfo.getCount(), 100);
}
// new node added but 1 child's output buffer is overutilized - so lockdown the tasks
nodeManager.addNode(CONNECTOR_ID, new InternalNode("other4", URI.create("http://127.0.0.4:14"), NodeVersion.UNKNOWN, false));
scheduleResult = scheduler.schedule();
assertEquals(scheduleResult.getBlockedReason().get(), SPLIT_QUEUES_FULL);
assertEquals(scheduleResult.getNewTasks().size(), 0);
assertEquals(scheduleResult.getSplitsScheduled(), 0);
}
use of io.trino.testing.TestingSplit in project trino by trinodb.
the class TestSourcePartitionedScheduler method testScheduleSlowSplitSource.
@Test
public void testScheduleSlowSplitSource() {
QueuedSplitSource queuedSplitSource = new QueuedSplitSource(TestingSplit::createRemoteSplit);
PlanFragment plan = createFragment();
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
StageExecution stage = createStageExecution(plan, nodeTaskMap);
StageScheduler scheduler = getSourcePartitionedScheduler(queuedSplitSource, stage, nodeManager, nodeTaskMap, 1, STAGE);
// schedule with no splits - will block
ScheduleResult scheduleResult = scheduler.schedule();
assertFalse(scheduleResult.isFinished());
assertFalse(scheduleResult.getBlocked().isDone());
assertEquals(scheduleResult.getNewTasks().size(), 0);
assertEquals(stage.getAllTasks().size(), 0);
queuedSplitSource.addSplits(1);
assertTrue(scheduleResult.getBlocked().isDone());
}
Aggregations