Examples with NodeTtl com.facebook.presto.spi.ttl.NodeTtl used on opensource projects

Example 1 with NodeTtl

use of com.facebook.presto.spi.ttl.NodeTtl in project presto by prestodb.

the class TestPercentileBasedClusterTtlProvider method testWithNonEmptyTtlList.

@Test
public void testWithNonEmptyTtlList() {
    List<NodeTtl> nodeTtls = ImmutableList.of(new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(10, 100))), new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(30, 100))));
    assertEquals(clusterTtlProvider.getClusterTtl(nodeTtls), new ConfidenceBasedTtlInfo(10, 100));
}

Example 2 with NodeTtl

use of com.facebook.presto.spi.ttl.NodeTtl in project presto by prestodb.

the class SimpleTtlNodeSelector method getEligibleNodes.

private List<InternalNode> getEligibleNodes(int limit, NodeMap nodeMap, List<RemoteTask> existingTasks) {
    Map<InternalNode, NodeTtl> nodeTtlInfo = nodeTtlFetcherManager.getAllTtls();
    Map<InternalNode, Optional<ConfidenceBasedTtlInfo>> ttlInfo = nodeTtlInfo.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue().getTtlInfo().stream().min(Comparator.comparing(ConfidenceBasedTtlInfo::getExpiryInstant))));
    Duration estimatedExecutionTimeRemaining = getEstimatedExecutionTimeRemaining();
    // Of the nodes on which already have existing tasks, pick only those whose TTL is enough
    List<InternalNode> existingEligibleNodes = existingTasks.stream().map(remoteTask -> nodeMap.getActiveNodesByNodeId().get(remoteTask.getNodeId())).filter(Objects::nonNull).filter(ttlInfo::containsKey).filter(node -> ttlInfo.get(node).isPresent()).filter(node -> isTtlEnough(ttlInfo.get(node).get(), estimatedExecutionTimeRemaining)).collect(toList());
    int alreadySelectedNodeCount = existingEligibleNodes.size();
    List<InternalNode> activeNodes = nodeMap.getActiveNodes();
    List<InternalNode> newEligibleNodes = filterNodesByTtl(activeNodes, ImmutableSet.copyOf(existingEligibleNodes), ttlInfo, estimatedExecutionTimeRemaining);
    if (alreadySelectedNodeCount < limit && newEligibleNodes.size() > 0) {
        List<InternalNode> moreNodes = selectNodes(limit - alreadySelectedNodeCount, new ResettableRandomizedIterator<>(newEligibleNodes));
        existingEligibleNodes.addAll(moreNodes);
    }
    verify(existingEligibleNodes.stream().allMatch(Objects::nonNull), "existingNodes list must not contain any nulls");
    return existingEligibleNodes;
}

Example 3 with NodeTtl

use of com.facebook.presto.spi.ttl.NodeTtl in project presto by prestodb.

the class TestNodeScheduler method testTtlAwareScheduling.

@Test
public void testTtlAwareScheduling() {
    InMemoryNodeManager nodeManager = new InMemoryNodeManager();
    InternalNode node1 = new InternalNode("other1", URI.create("http://127.0.0.1:11"), NodeVersion.UNKNOWN, false);
    InternalNode node2 = new InternalNode("other2", URI.create("http://127.0.0.1:12"), NodeVersion.UNKNOWN, false);
    InternalNode node3 = new InternalNode("other3", URI.create("http://127.0.0.1:13"), NodeVersion.UNKNOWN, false);
    List<InternalNode> nodes = ImmutableList.of(node1, node2, node3);
    nodeManager.addNode(CONNECTOR_ID, nodes);
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setMaxSplitsPerNode(20).setIncludeCoordinator(false).setMaxPendingSplitsPerTask(10);
    Instant currentInstant = Instant.now();
    NodeTtl ttl1 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(5, ChronoUnit.MINUTES).getEpochSecond(), 100)));
    NodeTtl ttl2 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(30, ChronoUnit.MINUTES).getEpochSecond(), 100)));
    NodeTtl ttl3 = new NodeTtl(ImmutableSet.of(new ConfidenceBasedTtlInfo(currentInstant.plus(2, ChronoUnit.HOURS).getEpochSecond(), 100)));
    Map<NodeInfo, NodeTtl> nodeToTtl = ImmutableMap.of(new NodeInfo(node1.getNodeIdentifier(), node1.getHost()), ttl1, new NodeInfo(node2.getNodeIdentifier(), node2.getHost()), ttl2, new NodeInfo(node3.getNodeIdentifier(), node3.getHost()), ttl3);
    ConfidenceBasedNodeTtlFetcherManager nodeTtlFetcherManager = new ConfidenceBasedNodeTtlFetcherManager(nodeManager, new NodeSchedulerConfig(), new NodeTtlFetcherManagerConfig());
    NodeTtlFetcherFactory nodeTtlFetcherFactory = new TestingNodeTtlFetcherFactory(nodeToTtl);
    nodeTtlFetcherManager.addNodeTtlFetcherFactory(nodeTtlFetcherFactory);
    nodeTtlFetcherManager.load(nodeTtlFetcherFactory.getName(), ImmutableMap.of());
    nodeTtlFetcherManager.refreshTtlInfo();
    TestingQueryManager queryManager = new TestingQueryManager();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, new NodeSelectionStats(), nodeSchedulerConfig, nodeTaskMap, nodeTtlFetcherManager, queryManager, new SimpleTtlNodeSelectorConfig());
    // Query is estimated to take 20 mins and has been executing for 3 mins, i.e, 17 mins left
    // So only node2 and node3 have enough TTL to run additional work
    Session session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(20, TimeUnit.MINUTES));
    NodeSelector nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(3, TimeUnit.MINUTES));
    assertEquals(ImmutableSet.copyOf(nodeSelector.selectRandomNodes(3)), ImmutableSet.of(node2, node3));
    // Query is estimated to take 1 hour and has been executing for 45 mins, i.e, 15 mins left
    // So only node2 and node3 have enough TTL to work on new splits
    session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(45, TimeUnit.MINUTES));
    Set<Split> splits = new HashSet<>();
    for (int i = 0; i < 2; i++) {
        splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    }
    Multimap<InternalNode, Split> assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.size(), 2);
    assertTrue(assignments.keySet().contains(node2));
    assertTrue(assignments.keySet().contains(node3));
    // Query is estimated to take 1 hour and has been executing for 20 mins, i.e, 40 mins left
    // So only node3 has enough TTL to work on new splits
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    TaskId taskId = new TaskId("test", 1, 0, 1);
    RemoteTask newRemoteTask = remoteTaskFactory.createTableScanTask(taskId, node2, ImmutableList.of(), nodeTaskMap.createTaskStatsTracker(node2, taskId));
    taskMap.put(node2, newRemoteTask);
    nodeTaskMap.addTask(node2, newRemoteTask);
    session = sessionWithTtlAwareSchedulingStrategyAndEstimatedExecutionTime(new Duration(1, TimeUnit.HOURS));
    nodeSelector = nodeScheduler.createNodeSelector(session, CONNECTOR_ID);
    queryManager.setExecutionTime(new Duration(20, TimeUnit.MINUTES));
    splits.clear();
    for (int i = 0; i < 2; i++) {
        splits.add(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    }
    assignments = nodeSelector.computeAssignments(splits, ImmutableList.copyOf(taskMap.values())).getAssignments();
    assertEquals(assignments.size(), 2);
    assertEquals(assignments.keySet().size(), 1);
    assertTrue(assignments.keySet().contains(node3));
}

Example 4 with NodeTtl

use of com.facebook.presto.spi.ttl.NodeTtl in project presto by prestodb.

the class SimpleTtlNodeSelector method selectRandomNodes.

@Override
public List<InternalNode> selectRandomNodes(int limit, Set<InternalNode> excludedNodes) {
    Map<InternalNode, NodeTtl> nodeTtlInfo = nodeTtlFetcherManager.getAllTtls();
    Map<InternalNode, Optional<ConfidenceBasedTtlInfo>> ttlInfo = nodeTtlInfo.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue().getTtlInfo().stream().min(Comparator.comparing(ConfidenceBasedTtlInfo::getExpiryInstant))));
    NodeMap nodeMap = this.nodeMap.get().get();
    List<InternalNode> activeNodes = nodeMap.getActiveNodes();
    Duration estimatedExecutionTimeRemaining = getEstimatedExecutionTimeRemaining();
    List<InternalNode> eligibleNodes = filterNodesByTtl(activeNodes, excludedNodes, ttlInfo, estimatedExecutionTimeRemaining);
    return selectNodes(limit, new ResettableRandomizedIterator<>(eligibleNodes));
}

Example 5 with NodeTtl

use of com.facebook.presto.spi.ttl.NodeTtl in project presto by prestodb.

the class ConfidenceBasedNodeTtlFetcherManager method getStaleTtlWorkerCount.

@Managed
public long getStaleTtlWorkerCount() {
    Duration staleDuration = nodeTtlFetcherManagerConfig.getStaleTtlThreshold();
    Instant staleInstant = Instant.now().minus(staleDuration.toMillis(), ChronoUnit.MILLIS);
    return nodeTtlMap.values().stream().filter(nodeTtl -> nodeTtl.getTtlPredictionInstant().isBefore(staleInstant)).count();
}