use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestNodeScheduler method testAffinityAssignmentWithConsistentHashingWithVirtualNodes.
@Test
public void testAffinityAssignmentWithConsistentHashingWithVirtualNodes() {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setNodeSelectionHashStrategy(CONSISTENT_HASHING).setMinVirtualNodeCount(5).setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
Set<Split> splits = new HashSet<>();
IntStream.range(0, 10).forEach(i -> splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestAffinitySplitRemote(i))));
InternalNode node1 = new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false);
InternalNode node2 = new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false);
InternalNode node3 = new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false);
InternalNode node4 = new InternalNode("other4", URI.create("http://127.0.0.1:14"), NodeVersion.UNKNOWN, false);
// In setup node 1-3 are added to node manager
// consistent hashing ring for nodes:
// entry0 (-1907319920): node2
// entry1 (-1028466245): node3
// entry2 ( -546736344): node2
// entry3 ( 1127574531): node3
// entry4 ( 1166245243): node1
// entry5 ( 2145381619): node1
SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
// hashing value for splits:
// 0: -1962219106 -> entry0 -> node2
// 1: 145569539 -> entry3 -> node3
// 2: -1599101205 -> entry1 -> node3
// 3: -165119218 -> entry3 -> node3
// 4: 1142216720 -> entry4 -> node1
// 5: 1347620135 -> entry5 -> node1
// 6: 1232195252 -> entry5 -> node1
// 7: 427886318 -> entry3 -> node3
// 8: 1469878697 -> entry5 -> node1
// 9: 296801082 -> entry3 -> node3
assertEquals(splitPlacementResult.getAssignments().keySet().size(), 3);
// node1: split 4, 5, 6, 8
Collection<ConnectorSplit> node1Splits = splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
// node2: split 0
Collection<Object> node2Splits = splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
// node3: split 1, 2, 3, 7, 9
Collection<Object> node3Splits = splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).collect(toImmutableSet());
// Scheduling the same splits on the same set of nodes should give the same assignment
nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
assertEquals(splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node1Splits);
assertEquals(splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node2Splits);
assertEquals(splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node3Splits);
// Adding node4, consistent hashing ring for nodes:
// entry0 (-1907319920): node2
// entry1 (-1616890413): node4
// entry2 (-1028466245): node3
// entry3 ( -546736344): node2
// entry4 ( 1127574531): node3
// entry5 ( 1166245243): node1
// entry6 ( 1691928386): node4
// entry7 ( 2145381619): node1
nodeManager.addNode(CONNECTOR_ID, node4);
nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
// hashing value for splits:
// 0: -1962219106 -> entry0 -> node2
// 1: 145569539 -> entry4 -> node3
// 2: -1599101205 -> entry2 -> node3
// 3: -165119218 -> entry4 -> node3
// 4: 1142216720 -> entry5 -> node1
// 5: 1347620135 -> entry6 -> node4
// 6: 1232195252 -> entry6 -> node4
// 7: 427886318 -> entry4 -> node3
// 8: 1469878697 -> entry6 -> node4
// 9: 296801082 -> entry4 -> node3
assertEquals(splitPlacementResult.getAssignments().keySet().size(), 4);
assertEquals(splitPlacementResult.getAssignments().get(node1).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(4));
assertEquals(splitPlacementResult.getAssignments().get(node2).stream().map(Split::getConnectorSplit).collect(toImmutableSet()), node2Splits);
assertEquals(splitPlacementResult.getAssignments().get(node3).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(1, 2, 3, 7, 9));
assertEquals(splitPlacementResult.getAssignments().get(node4).stream().map(Split::getConnectorSplit).map(ConnectorSplit::getSplitIdentifier).collect(toImmutableSet()), ImmutableSet.of(5, 6, 8));
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestNodeScheduler method testHardAffinityAssignment.
@Test
public void testHardAffinityAssignment() {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 3);
Set<Split> splits = new HashSet<>();
// Adding one more split (1 % 3 = 1), 1 splits will be distributed to 1 nodes
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestHardAffinitySplitRemote()));
SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
for (Split split : splitPlacementResult.getAssignments().values()) {
assertTrue(split.getSplitContext().isCacheable());
}
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestSimpleTtlNodeSelectorConfig method testExplicitPropertyMappings.
@Test
public void testExplicitPropertyMappings() {
Map<String, String> properties = new ImmutableMap.Builder<String, String>().put("simple-ttl-node-selector.use-default-execution-time-estimate-as-fallback", "true").put("simple-ttl-node-selector.default-execution-time-estimate", "1h").build();
SimpleTtlNodeSelectorConfig expected = new SimpleTtlNodeSelectorConfig().setUseDefaultExecutionTimeEstimateAsFallback(true).setDefaultExecutionTimeEstimate(new Duration(1, TimeUnit.HOURS));
ConfigAssertions.assertFullMapping(properties, expected);
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestSourcePartitionedScheduler method getSourcePartitionedScheduler.
private static StageScheduler getSourcePartitionedScheduler(ConnectorSplitSource connectorSplitSource, SqlStageExecution stage, InternalNodeManager nodeManager, NodeTaskMap nodeTaskMap, int splitBatchSize) {
NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setIncludeCoordinator(false).setMaxSplitsPerNode(20).setMaxPendingSplitsPerTask(0);
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
SplitSource splitSource = new ConnectorAwareSplitSource(CONNECTOR_ID, TestingTransactionHandle.create(), connectorSplitSource);
SplitPlacementPolicy placementPolicy = new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(TestingSession.testSessionBuilder().build(), splitSource.getConnectorId()), stage::getAllTasks);
return newSourcePartitionedSchedulerAsStageScheduler(stage, TABLE_SCAN_NODE_ID, splitSource, placementPolicy, splitBatchSize);
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestSourcePartitionedScheduler method testNoNodes.
@Test
public void testNoNodes() {
try {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
InMemoryNodeManager nodeManager = new InMemoryNodeManager();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(false), nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
SubPlan plan = createPlan();
SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
StageScheduler scheduler = newSourcePartitionedSchedulerAsStageScheduler(stage, TABLE_SCAN_NODE_ID, new ConnectorAwareSplitSource(CONNECTOR_ID, TestingTransactionHandle.create(), createFixedSplitSource(20, TestingSplit::createRemoteSplit)), new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(TestingSession.testSessionBuilder().build(), CONNECTOR_ID), stage::getAllTasks), 2);
scheduler.schedule();
fail("expected PrestoException");
} catch (PrestoException e) {
assertEquals(e.getErrorCode(), NO_NODES_AVAILABLE.toErrorCode());
}
}
Aggregations