use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.
the class TestNodeScheduler method testMaxUnacknowledgedSplitsPerTask.
@Test
public void testMaxUnacknowledgedSplitsPerTask() {
int maxUnacknowledgedSplitsPerTask = 5;
nodeSelector = nodeScheduler.createNodeSelector(sessionWithMaxUnacknowledgedSplitsPerTask(maxUnacknowledgedSplitsPerTask), CONNECTOR_ID, Integer.MAX_VALUE);
TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
ImmutableList.Builder<Split> initialSplits = ImmutableList.builder();
for (int i = 0; i < maxUnacknowledgedSplitsPerTask; i++) {
initialSplits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
}
List<InternalNode> nodes = new ArrayList<>();
List<MockRemoteTaskFactory.MockRemoteTask> tasks = new ArrayList<>();
MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
int counter = 1;
for (InternalNode node : nodeManager.getActiveConnectorNodes(CONNECTOR_ID)) {
// Max out number of unacknowledged splits on each task
TaskId taskId = new TaskId("test", 1, 0, counter);
counter++;
MockRemoteTaskFactory.MockRemoteTask remoteTask = remoteTaskFactory.createTableScanTask(taskId, node, initialSplits.build(), nodeTaskMap.createTaskStatsTracker(node, taskId));
nodeTaskMap.addTask(node, remoteTask);
remoteTask.setMaxUnacknowledgedSplits(maxUnacknowledgedSplitsPerTask);
remoteTask.setUnacknowledgedSplits(maxUnacknowledgedSplitsPerTask);
nodes.add(node);
tasks.add(remoteTask);
}
// One split per node
Set<Split> splits = new HashSet<>();
for (int i = 0; i < nodes.size(); i++) {
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
}
SplitPlacementResult splitPlacements = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(tasks));
// No splits should have been placed, max unacknowledged was already reached
assertEquals(splitPlacements.getAssignments().size(), 0);
assertFalse(splitPlacements.getBlocked().isDone());
// Unblock one task
MockRemoteTaskFactory.MockRemoteTask taskOne = tasks.get(0);
taskOne.finishSplits(1);
taskOne.setUnacknowledgedSplits(taskOne.getUnacknowledgedPartitionedSplitCount() - 1);
assertTrue(splitPlacements.getBlocked().isDone());
// Attempt to schedule again, only the node with the unblocked task should be chosen
splitPlacements = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(tasks));
assertEquals(splitPlacements.getAssignments().size(), 1);
assertTrue(splitPlacements.getAssignments().keySet().contains(nodes.get(0)));
// Make the first node appear to have no splits, unacknowledged splits alone should force the splits to be spread across nodes
taskOne.clearSplits();
// Give all tasks with room for 1 unacknowledged split
tasks.forEach(task -> task.setUnacknowledgedSplits(maxUnacknowledgedSplitsPerTask - 1));
splitPlacements = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(tasks));
// One split placed on each node
assertEquals(splitPlacements.getAssignments().size(), nodes.size());
assertTrue(splitPlacements.getAssignments().keySet().containsAll(nodes));
}
use of com.facebook.presto.testing.TestingTransactionHandle in project presto by prestodb.
the class TestNodeScheduler method testHardAffinityAssignment.
@Test
public void testHardAffinityAssignment() {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
Set<Split> splits = new HashSet<>();
// Adding one more split (1 % 3 = 1), 1 splits will be distributed to 1 nodes
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
for (Split split : splitPlacementResult.getAssignments().values()) {
assertTrue(split.getSplitContext().isCacheable());
}
}
Aggregations