Examples with InternalNode com.facebook.presto.metadata.InternalNode used on opensource projects

Example 51 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestClusterSizeMonitor method addWorker.

private void addWorker(InMemoryNodeManager nodeManager) {
    String identifier = "worker/" + numWorkers.incrementAndGet();
    nodeManager.addNode(CONNECTOR_ID, new InternalNode(identifier, URI.create("localhost/" + identifier), new NodeVersion("1"), false));
}

Example 52 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class TestNodeScheduler method testMemoryUsage.

@Test
public void testMemoryUsage() {
    MockRemoteTaskFactory remoteTaskFactory = new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor);
    InternalNode chosenNode = Iterables.get(nodeManager.getActiveConnectorNodes(CONNECTOR_ID), 0);
    TaskId taskId1 = new TaskId("test", 1, 0, 1);
    List<Split> splits = ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()), new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote()));
    RemoteTask remoteTask1 = remoteTaskFactory.createTableScanTask(taskId1, chosenNode, splits, nodeTaskMap.createTaskStatsTracker(chosenNode, taskId1));
    TaskId taskId2 = new TaskId("test", 1, 0, 2);
    RemoteTask remoteTask2 = remoteTaskFactory.createTableScanTask(taskId2, chosenNode, ImmutableList.of(new Split(CONNECTOR_ID, TestingTransactionHandle.create(), new TestSplitRemote())), nodeTaskMap.createTaskStatsTracker(chosenNode, taskId2));
    nodeTaskMap.addTask(chosenNode, remoteTask1);
    nodeTaskMap.addTask(chosenNode, remoteTask2);
    remoteTask2.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), splits).build());
    assertEquals(nodeTaskMap.getNodeTotalMemoryUsageInBytes(chosenNode), 200);
    remoteTask1.addSplits(ImmutableMultimap.<PlanNodeId, Split>builder().putAll(new PlanNodeId("sourceId"), splits).build());
    assertEquals(nodeTaskMap.getNodeTotalMemoryUsageInBytes(chosenNode), 200);
    remoteTask1.abort();
    assertEquals(nodeTaskMap.getNodeTotalMemoryUsageInBytes(chosenNode), 100);
    remoteTask2.abort();
    assertEquals(nodeTaskMap.getNodeTotalMemoryUsageInBytes(chosenNode), 0);
}

Example 53 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class DynamicLifespanScheduler method scheduleInitial.

@Override
public void scheduleInitial(SourceScheduler scheduler) {
    checkState(!initialScheduled, "Initial scheduling happened before scheduleInitial is called");
    int driverGroupsScheduledPerTask = 0;
    synchronized (this) {
        while (!noPreferenceDriverGroups.isEmpty() || !nodeToPreferredDriverGroups.isEmpty()) {
            for (int taskId = 0; taskId < nodeByTaskId.size() && (!noPreferenceDriverGroups.isEmpty() || !nodeToPreferredDriverGroups.isEmpty()); taskId++) {
                InternalNode node = nodeByTaskId.get(taskId);
                OptionalInt driverGroupId = getNextDriverGroup(node);
                if (!driverGroupId.isPresent()) {
                    continue;
                }
                scheduler.startLifespan(Lifespan.driverGroup(driverGroupId.getAsInt()), partitionHandles.get(driverGroupId.getAsInt()));
                taskByDriverGroup[driverGroupId.getAsInt()] = taskId;
                runningDriverGroupIdsByTask[taskId].add(driverGroupId.getAsInt());
            }
            driverGroupsScheduledPerTask++;
            if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduledPerTask == concurrentLifespansPerTask.getAsInt()) {
                break;
            }
        }
        initialScheduled = true;
    }
}

Example 54 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class SimpleTtlNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks) {
    boolean isNodeSelectionStrategyNoPreference = splits.stream().allMatch(split -> split.getNodeSelectionStrategy() == NodeSelectionStrategy.NO_PREFERENCE);
    // Current NodeSelectionStrategy support is limited to NO_PREFERENCE
    if (!isNodeSelectionStrategyNoPreference) {
        return simpleNodeSelector.computeAssignments(splits, existingTasks);
    }
    ImmutableMultimap.Builder<InternalNode, Split> assignment = ImmutableMultimap.builder();
    NodeMap nodeMap = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    List<InternalNode> eligibleNodes = getEligibleNodes(maxTasksPerStage, nodeMap, existingTasks);
    NodeSelection randomNodeSelection = new RandomNodeSelection(eligibleNodes, minCandidates);
    boolean splitWaitingForAnyNode = false;
    OptionalInt preferredNodeCount = OptionalInt.empty();
    for (Split split : splits) {
        if (split.getNodeSelectionStrategy() != NodeSelectionStrategy.NO_PREFERENCE) {
            throw new PrestoException(NODE_SELECTION_NOT_SUPPORTED, format("Unsupported node selection strategy for TTL scheduling: %s", split.getNodeSelectionStrategy()));
        }
        List<InternalNode> candidateNodes = randomNodeSelection.pickNodes(split);
        if (candidateNodes.isEmpty()) {
            log.warn("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getActiveNodes());
            throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
        }
        SplitWeight splitWeight = split.getSplitWeight();
        Optional<InternalNodeInfo> chosenNodeInfo = simpleNodeSelector.chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getTotalSplitsWeight, preferredNodeCount, maxSplitsWeightPerNode, assignmentStats);
        if (!chosenNodeInfo.isPresent()) {
            chosenNodeInfo = simpleNodeSelector.chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getQueuedSplitsWeightForStage, preferredNodeCount, maxPendingSplitsWeightPerTask, assignmentStats);
        }
        if (chosenNodeInfo.isPresent()) {
            split = new Split(split.getConnectorId(), split.getTransactionHandle(), split.getConnectorSplit(), split.getLifespan(), new SplitContext(chosenNodeInfo.get().isCacheable()));
            InternalNode chosenNode = chosenNodeInfo.get().getInternalNode();
            assignment.put(chosenNode, split);
            assignmentStats.addAssignedSplit(chosenNode, splitWeight);
        } else {
            splitWaitingForAnyNode = true;
        }
    }
    ListenableFuture<?> blocked = splitWaitingForAnyNode ? toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingSplitsWeightPerTask)) : immediateFuture(null);
    return new SplitPlacementResult(blocked, assignment.build());
}

Example 55 with InternalNode

use of com.facebook.presto.metadata.InternalNode in project presto by prestodb.

the class SimpleTtlNodeSelector method selectRandomNodes.

@Override
public List<InternalNode> selectRandomNodes(int limit, Set<InternalNode> excludedNodes) {
    Map<InternalNode, NodeTtl> nodeTtlInfo = nodeTtlFetcherManager.getAllTtls();
    Map<InternalNode, Optional<ConfidenceBasedTtlInfo>> ttlInfo = nodeTtlInfo.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue().getTtlInfo().stream().min(Comparator.comparing(ConfidenceBasedTtlInfo::getExpiryInstant))));
    NodeMap nodeMap = this.nodeMap.get().get();
    List<InternalNode> activeNodes = nodeMap.getActiveNodes();
    Duration estimatedExecutionTimeRemaining = getEstimatedExecutionTimeRemaining();
    List<InternalNode> eligibleNodes = filterNodesByTtl(activeNodes, excludedNodes, ttlInfo, estimatedExecutionTimeRemaining);
    return selectNodes(limit, new ResettableRandomizedIterator<>(eligibleNodes));
}