Examples with InternalNode io.trino.metadata.InternalNode used on opensource projects

Example 6 with InternalNode

use of io.trino.metadata.InternalNode in project trino by trinodb.

the class TopologyAwareNodeSelector method bestNodeSplitCount.

@Nullable
private InternalNode bestNodeSplitCount(SplitWeight splitWeight, Iterator<InternalNode> candidates, int minCandidatesWhenFull, long maxPendingSplitsWeightPerTask, NodeAssignmentStats assignmentStats) {
    InternalNode bestQueueNotFull = null;
    long minWeight = Long.MAX_VALUE;
    int fullCandidatesConsidered = 0;
    while (candidates.hasNext() && (fullCandidatesConsidered < minCandidatesWhenFull || bestQueueNotFull == null)) {
        InternalNode node = candidates.next();
        if (assignmentStats.getUnacknowledgedSplitCountForStage(node) >= maxUnacknowledgedSplitsPerTask) {
            fullCandidatesConsidered++;
            continue;
        }
        if (canAssignSplitBasedOnWeight(assignmentStats.getTotalSplitsWeight(node), maxSplitsWeightPerNode, splitWeight)) {
            return node;
        }
        fullCandidatesConsidered++;
        long taskQueuedWeight = assignmentStats.getQueuedSplitsWeightForStage(node);
        if (taskQueuedWeight < minWeight && canAssignSplitBasedOnWeight(taskQueuedWeight, maxPendingSplitsWeightPerTask, splitWeight)) {
            minWeight = taskQueuedWeight;
            bestQueueNotFull = node;
        }
    }
    return bestQueueNotFull;
}

Example 7 with InternalNode

use of io.trino.metadata.InternalNode in project trino by trinodb.

the class TopologyAwareNodeSelectorFactory method createNodeMap.

private NodeMap createNodeMap(Optional<CatalogName> catalogName) {
    Set<InternalNode> nodes = catalogName.map(nodeManager::getActiveConnectorNodes).orElseGet(() -> nodeManager.getNodes(ACTIVE));
    Set<String> coordinatorNodeIds = nodeManager.getCoordinators().stream().map(InternalNode::getNodeIdentifier).collect(toImmutableSet());
    ImmutableSetMultimap.Builder<HostAddress, InternalNode> byHostAndPort = ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<InetAddress, InternalNode> byHost = ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<NetworkLocation, InternalNode> workersByNetworkPath = ImmutableSetMultimap.builder();
    for (InternalNode node : nodes) {
        if (includeCoordinator || !coordinatorNodeIds.contains(node.getNodeIdentifier())) {
            NetworkLocation location = networkTopology.locate(node.getHostAndPort());
            for (int i = 0; i <= location.getSegments().size(); i++) {
                workersByNetworkPath.put(location.subLocation(0, i), node);
            }
        }
        try {
            byHostAndPort.put(node.getHostAndPort(), node);
            byHost.put(node.getInternalAddress(), node);
        } catch (UnknownHostException e) {
            if (markInaccessibleNode(node)) {
                LOG.warn(e, "Unable to resolve host name for node: %s", node);
            }
        }
    }
    return new NodeMap(byHostAndPort.build(), byHost.build(), workersByNetworkPath.build(), coordinatorNodeIds);
}

Example 8 with InternalNode

use of io.trino.metadata.InternalNode in project trino by trinodb.

the class UniformNodeSelector method equateDistribution.

/**
 * The method tries to make the distribution of splits more uniform. All nodes are arranged into a maxHeap and a minHeap
 * based on the number of splits that are assigned to them. Splits are redistributed, one at a time, from a maxNode to a
 * minNode until we have as uniform a distribution as possible.
 *
 * @param assignment the node-splits multimap after the first and the second stage
 * @param assignmentStats required to obtain info regarding splits assigned to a node outside the current batch of assignment
 * @param nodeMap to get a list of all nodes to which splits can be assigned
 */
private void equateDistribution(Multimap<InternalNode, Split> assignment, NodeAssignmentStats assignmentStats, NodeMap nodeMap, boolean includeCoordinator) {
    if (assignment.isEmpty()) {
        return;
    }
    Collection<InternalNode> allNodes = nodeMap.getNodesByHostAndPort().values().stream().filter(node -> includeCoordinator || !nodeMap.getCoordinatorNodeIds().contains(node.getNodeIdentifier())).collect(toImmutableList());
    if (allNodes.size() < 2) {
        return;
    }
    IndexedPriorityQueue<InternalNode> maxNodes = new IndexedPriorityQueue<>();
    for (InternalNode node : assignment.keySet()) {
        maxNodes.addOrUpdate(node, assignmentStats.getTotalSplitsWeight(node));
    }
    IndexedPriorityQueue<InternalNode> minNodes = new IndexedPriorityQueue<>();
    for (InternalNode node : allNodes) {
        minNodes.addOrUpdate(node, Long.MAX_VALUE - assignmentStats.getTotalSplitsWeight(node));
    }
    while (true) {
        if (maxNodes.isEmpty()) {
            return;
        }
        // fetch min and max node
        InternalNode maxNode = maxNodes.poll();
        InternalNode minNode = minNodes.poll();
        // misassigned splits greatly (in absolute values).
        if (assignmentStats.getTotalSplitsWeight(maxNode) - assignmentStats.getTotalSplitsWeight(minNode) <= SplitWeight.rawValueForStandardSplitCount(5)) {
            return;
        }
        // move split from max to min
        Split redistributed = redistributeSplit(assignment, maxNode, minNode, nodeMap.getNodesByHost());
        assignmentStats.removeAssignedSplit(maxNode, redistributed.getSplitWeight());
        assignmentStats.addAssignedSplit(minNode, redistributed.getSplitWeight());
        // add max back into maxNodes only if it still has assignments
        if (assignment.containsKey(maxNode)) {
            maxNodes.addOrUpdate(maxNode, assignmentStats.getTotalSplitsWeight(maxNode));
        }
        // Add or update both the Priority Queues with the updated node priorities
        maxNodes.addOrUpdate(minNode, assignmentStats.getTotalSplitsWeight(minNode));
        minNodes.addOrUpdate(minNode, Long.MAX_VALUE - assignmentStats.getTotalSplitsWeight(minNode));
        minNodes.addOrUpdate(maxNode, Long.MAX_VALUE - assignmentStats.getTotalSplitsWeight(maxNode));
    }
}

Example 9 with InternalNode

use of io.trino.metadata.InternalNode in project trino by trinodb.

the class UniformNodeSelectorFactory method createNodeMap.

private NodeMap createNodeMap(Optional<CatalogName> catalogName) {
    Set<InternalNode> nodes = catalogName.map(nodeManager::getActiveConnectorNodes).orElseGet(() -> nodeManager.getNodes(ACTIVE));
    Set<String> coordinatorNodeIds = nodeManager.getCoordinators().stream().map(InternalNode::getNodeIdentifier).collect(toImmutableSet());
    ImmutableSetMultimap.Builder<HostAddress, InternalNode> byHostAndPort = ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<InetAddress, InternalNode> byHost = ImmutableSetMultimap.builder();
    for (InternalNode node : nodes) {
        try {
            byHostAndPort.put(node.getHostAndPort(), node);
            byHost.put(node.getInternalAddress(), node);
        } catch (UnknownHostException e) {
            if (markInaccessibleNode(node)) {
                LOG.warn(e, "Unable to resolve host name for node: %s", node);
            }
        }
    }
    return new NodeMap(byHostAndPort.build(), byHost.build(), ImmutableSetMultimap.of(), coordinatorNodeIds);
}

Example 10 with InternalNode

use of io.trino.metadata.InternalNode in project trino by trinodb.

the class DynamicLifespanScheduler method schedule.

@Override
public SettableFuture<Void> schedule(SourceScheduler scheduler) {
    // Return a new future even if newDriverGroupReady has not finished.
    // Returning the same SettableFuture instance could lead to ListenableFuture retaining too many listener objects.
    checkState(initialScheduled);
    List<Lifespan> recentlyCompletedDriverGroups;
    synchronized (this) {
        recentlyCompletedDriverGroups = ImmutableList.copyOf(this.recentlyCompletedDriverGroups);
        this.recentlyCompletedDriverGroups.clear();
        newDriverGroupReady = SettableFuture.create();
    }
    for (Lifespan driverGroup : recentlyCompletedDriverGroups) {
        if (!driverGroups.hasNext()) {
            break;
        }
        int driverGroupId = driverGroups.nextInt();
        InternalNode nodeForCompletedDriverGroup = bucketNodeMap.getAssignedNode(driverGroup.getId()).orElseThrow(IllegalStateException::new);
        bucketNodeMap.assignBucketToNode(driverGroupId, nodeForCompletedDriverGroup);
        scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
    }
    if (!driverGroups.hasNext()) {
        scheduler.noMoreLifespans();
    }
    return newDriverGroupReady;
}