use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
MetadataManager metadata = MetadataManager.createTestMetadataManager();
PageSourceManager pageSourceManager = new PageSourceManager();
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager();
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager, new NodeSelectionStats());
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
return new LocalExecutionPlanner(metadata, Optional.empty(), pageSourceManager, new IndexManager(), partitioningProviderManager, nodePartitioningManager, new PageSinkManager(), new ConnectorMetadataUpdaterManager(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new MemoryManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new BlockEncodingManager(), new PagesIndex.TestingFactory(false), new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()), new LookupJoinOperators(), new OrderingCompiler(), jsonCodec(TableCommitContext.class), new RowExpressionDeterminismEvaluator(metadata), new NoOpFragmentResultCacheManager(), new ObjectMapper(), (session) -> {
throw new UnsupportedOperationException();
});
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestCostCalculator method setUp.
@BeforeClass
public void setUp() {
TaskCountEstimator taskCountEstimator = new TaskCountEstimator(() -> NUMBER_OF_NODES);
costCalculatorUsingExchanges = new CostCalculatorUsingExchanges(taskCountEstimator);
costCalculatorWithEstimatedExchanges = new CostCalculatorWithEstimatedExchanges(costCalculatorUsingExchanges, taskCountEstimator);
session = testSessionBuilder().setCatalog("tpch").build();
CatalogManager catalogManager = new CatalogManager();
catalogManager.registerCatalog(createBogusTestingCatalog("tpch"));
transactionManager = createTestTransactionManager(catalogManager);
metadata = createTestMetadataManager(transactionManager, new FeaturesConfig());
finalizerService = new FinalizerService();
finalizerService.start();
nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSelectionStats(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
PartitioningProviderManager partitioningProviderManager = new PartitioningProviderManager();
nodePartitioningManager = new NodePartitioningManager(nodeScheduler, partitioningProviderManager, new NodeSelectionStats());
planFragmenter = new PlanFragmenter(metadata, nodePartitioningManager, new QueryManagerConfig(), new SqlParser(), new FeaturesConfig());
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestNodeScheduler method testTtlAwareScheduling.
@Test
public void testTtlAwareScheduling() {
InMemoryNodeManager nodeManager = new InMemoryNodeManager();
InternalNode node1 = new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false);
InternalNode node2 = new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false);
InternalNode node3 = new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false);
List<InternalNode> nodes = ImmutableList.of(node1, node2, node3);
nodeManager.addNode(CONNECTOR_ID, nodes);
NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
Instant currentInstant = Instant.now();
NodeTtl ttl1 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(5, ChronoUnit.MINUTES).getEpochSecond(), 100)));
NodeTtl ttl2 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(30, ChronoUnit.MINUTES).getEpochSecond(), 100)));
NodeTtl ttl3 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(2, ChronoUnit.HOURS).getEpochSecond(), 100)));
Map<NodeInfo, NodeTtl> nodeToTtl = ImmutableMap.of(new NodeInfo(node1.getNodeIdentifier(), node1.getHost()), ttl1, new NodeInfo(node2.getNodeIdentifier(), node2.getHost()), ttl2, new NodeInfo(node3.getNodeIdentifier(), node3.getHost()), ttl3);
ConfidenceBasedNodeTtlFetcherManager nodeTtlFetcherManager = new ConfidenceBasedNodeTtlFetcherManager(nodeManager, new NodeSchedulerConfig(), new NodeTtlFetcherManagerConfig());
NodeTtlFetcherFactory nodeTtlFetcherFactory = new TestingNodeTtlFetcherFactory(nodeToTtl);
nodeTtlFetcherManager.addNodeTtlFetcherFactory(nodeTtlFetcherFactory);
nodeTtlFetcherManager.load(nodeTtlFetcherFactory.getName(), ImmutableMap.of());
nodeTtlFetcherManager.refreshTtlInfo();
TestingQueryManager queryManager = new TestingQueryManager();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, nodeTtlFetcherManager, queryManager, new SimpleTtlNodeSelectorConfig());
// Query is estimated to take 20 mins and has been executing for 3 mins, i.e, 17 mins left
// So only node2 and node3 have enough TTL to run additional work
Session session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(20, TimeUnit.MINUTES));
NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
queryManager.setExecutionTime(new Duration(3, TimeUnit.MINUTES));
assertEquals(ImmutableSet.copyOf(nodeSelector.selectRandomNodes(3)), ImmutableSet.of(node2, node3));
// Query is estimated to take 1 hour and has been executing for 45 mins, i.e, 15 mins left
// So only node2 and node3 have enough TTL to work on new splits
session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
queryManager.setExecutionTime(new Duration(45, TimeUnit.MINUTES));
Set<Split> splits = new HashSet<>();
for (int i = 0; i < 2; i++) {
splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
}
Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
assertEquals(assignments.size(), 2);
assertTrue(assignments.keySet().contains(node2));
assertTrue(assignments.keySet().contains(node3));
// Query is estimated to take 1 hour and has been executing for 20 mins, i.e, 40 mins left
// So only node3 has enough TTL to work on new splits
MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
TaskId taskId = new TaskId("test", 1, 0, 1);
RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, node2, ImmutableList.of(), nodeTaskMap.createTaskStatsTracker(node2, taskId));
taskMap.put(node2, newRemoteTask);
nodeTaskMap.addTask(node2, newRemoteTask);
session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
queryManager.setExecutionTime(new Duration(20, TimeUnit.MINUTES));
splits.clear();
for (int i = 0; i < 2; i++) {
splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
}
assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
assertEquals(assignments.size(), 2);
assertEquals(assignments.keySet().size(), 1);
assertTrue(assignments.keySet().contains(node3));
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestNodeScheduler method testAffinityAssignmentNotSupported.
@Test
public void testAffinityAssignmentNotSupported() {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
LegacyNetworkTopology legacyNetworkTopology = new LegacyNetworkTopology();
NodeScheduler nodeScheduler = new NodeScheduler(new NetworkLocationCache(legacyNetworkTopology), legacyNetworkTopology, nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new Duration(0, SECONDS), new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 2);
Set<Split> splits = new HashSet<>();
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromString("127.0.0.1:10"))));
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote(HostAddress.fromString("127.0.0.1:10"))));
SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
Set<InternalNode> internalNodes = splitPlacementResult.getAssignments().keySet();
// Split doesn't support affinity schedule, fall back to random schedule
assertEquals(internalNodes.size(), 2);
}
use of com.facebook.presto.execution.scheduler.nodeSelection.SimpleTtlNodeSelectorConfig in project presto by prestodb.
the class TestNodeScheduler method testMaxTasksPerStageWittLimit.
@Test
public void testMaxTasksPerStageWittLimit() {
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
TestingTransactionHandle transactionHandle = TestingTransactionHandle.create();
NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, new ThrowingNodeTtlFetcherManager(), new NoOpQueryManager(), new SimpleTtlNodeSelectorConfig());
NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID, 2);
Set<Split> splits = new HashSet<>();
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
SplitPlacementResult splitPlacementResult = nodeSelector.computeAssignments(splits, ImmutableList.of());
Set<InternalNode> internalNodes = splitPlacementResult.getAssignments().keySet();
assertEquals(internalNodes.size(), 1);
// adding one more split. Total 2
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult));
Set<InternalNode> internalNodesSecondCall = splitPlacementResult.getAssignments().keySet();
assertEquals(internalNodesSecondCall.size(), 2);
assertTrue(internalNodesSecondCall.containsAll(internalNodes));
// adding one more split. Total 3
splits.add(new Split(CONNECTOR_ID, transactionHandle, new TestSplitRemote()));
splitPlacementResult = nodeSelector.computeAssignments(splits, getRemoteTableScanTask(splitPlacementResult));
assertEquals(splitPlacementResult.getAssignments().keySet().size(), 2);
assertEquals(splitPlacementResult.getAssignments().keySet(), internalNodesSecondCall);
}
Aggregations