Examples with InternalNode io.prestosql.metadata.InternalNode used on opensource projects

Example 21 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class NodeScheduler method selectExactNodes.

public static List<InternalNode> selectExactNodes(NodeMap nodeMap, List<HostAddress> hosts, boolean includeCoordinator) {
    Set<InternalNode> chosen = new LinkedHashSet<>();
    for (HostAddress host : hosts) {
        nodeMap.getNodesByHostAndPort().get(host).stream().filter(InternalNode::isWorker).forEach(chosen::add);
        InetAddress address;
        try {
            address = host.toInetAddress();
        } catch (UnknownHostException e) {
            // skip hosts that don't resolve
            continue;
        }
        // consider a split with a host without a port as being accessible by all nodes in that host
        if (!host.hasPort()) {
            nodeMap.getNodesByHost().get(address).stream().filter(InternalNode::isWorker).forEach(chosen::add);
        }
    }
    // if the chosen set is empty and the host is the coordinator, force pick the coordinator
    if (chosen.isEmpty() && !includeCoordinator) {
        for (HostAddress host : hosts) {
            // In the code below, before calling `chosen::add`, it could have been checked that
            // `coordinatorIds.contains(node.getNodeIdentifier())`. But checking the condition isn't necessary
            // because every node satisfies it. Otherwise, `chosen` wouldn't have been empty.
            chosen.addAll(nodeMap.getNodesByHostAndPort().get(host));
            InetAddress address;
            try {
                address = host.toInetAddress();
            } catch (UnknownHostException e) {
                // skip hosts that don't resolve
                continue;
            }
            // consider a split with a host without a port as being accessible by all nodes in that host
            if (!host.hasPort()) {
                chosen.addAll(nodeMap.getNodesByHost().get(address));
            }
        }
    }
    return ImmutableList.copyOf(chosen);
}

Example 22 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class NodeScheduler method selectDistributionNodes.

public static SplitPlacementResult selectDistributionNodes(NodeMap nodeMap, NodeTaskMap nodeTaskMap, int maxSplitsPerNode, int maxPendingSplitsPerTask, Set<Split> splits, List<RemoteTask> existingTasks, BucketNodeMap bucketNodeMap) {
    Multimap<InternalNode, Split> assignments = HashMultimap.create();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    Set<InternalNode> blockedNodes = new HashSet<>();
    for (Split split : splits) {
        // node placement is forced by the bucket to node map
        InternalNode node = bucketNodeMap.getAssignedNode(split).get();
        // if node is full, don't schedule now, which will push back on the scheduling of splits
        if (assignmentStats.getTotalSplitCount(node) < maxSplitsPerNode || assignmentStats.getQueuedSplitCountForStage(node) < maxPendingSplitsPerTask) {
            assignments.put(node, split);
            assignmentStats.addAssignedSplit(node);
        } else {
            blockedNodes.add(node);
        }
    }
    ListenableFuture<?> blocked = toWhenHasSplitQueueSpaceFuture(blockedNodes, existingTasks, calculateLowWatermark(maxPendingSplitsPerTask));
    return new SplitPlacementResult(blocked, ImmutableMultimap.copyOf(assignments));
}

Example 23 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class SimpleFixedNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks, Optional<SqlStageExecution> stage) {
    Multimap<InternalNode, Split> assignment = HashMultimap.create();
    NodeMap nodeMap = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    List<InternalNode> candidateNodes = new ArrayList<>();
    if (!stage.isPresent()) {
        log.error("Cant schedule as stage missing");
        throw new PrestoException(GENERIC_INTERNAL_ERROR, "stage is empty");
    }
    PlanNodeId planNodeId = stage.get().getFragment().getFeederCTEParentId().get();
    // if still feeder has not been scheduled then no point in scheduling this also
    if (!feederScheduledNodes.containsKey(planNodeId)) {
        return new SplitPlacementResult(immediateFuture(null), assignment);
    }
    // Find max number of splits consumer can schedule in current cycle.
    int maxSplitsToSchedule = feederScheduledNodes.get(planNodeId).getSplitCount() - consumedNodes.getSplitCount();
    // find list of nodes where still consumer has not been scheduled.
    if (feederScheduledNodes.get(planNodeId).getAssignedNodes().equals(consumedNodes.getAssignedNodes())) {
        candidateNodes = new ArrayList<>(consumedNodes.getAssignedNodes());
    } else {
        for (InternalNode node : feederScheduledNodes.get(planNodeId).getAssignedNodes()) {
            if (!consumedNodes.getAssignedNodes().contains(node)) {
                candidateNodes.add(node);
                consumedNodes.getAssignedNodes().add(node);
            }
        }
    }
    // schedule derived number of splits on derived list of nodes.
    // It is expected that splits count should be at-least equal to number of nodes so that each node gets at-least
    // one split.
    int index = 0;
    int totalNodes = candidateNodes.size();
    for (Split split : Iterables.limit(splits, maxSplitsToSchedule)) {
        InternalNode chosenNode = candidateNodes.get(index % totalNodes);
        assignment.put(chosenNode, split);
        assignmentStats.addAssignedSplit(chosenNode);
        index++;
    }
    consumedNodes.updateSplitCount(maxSplitsToSchedule);
    return new SplitPlacementResult(immediateFuture(null), assignment);
}

Example 24 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class FixedLifespanScheduler method scheduleInitial.

@Override
public void scheduleInitial(SourceScheduler scheduler) {
    checkState(!initialScheduled);
    initialScheduled = true;
    for (Map.Entry<InternalNode, IntListIterator> entry : nodeToDriverGroupsMap.entrySet()) {
        IntListIterator driverGroupsIterator = entry.getValue();
        int driverGroupsScheduled = 0;
        while (driverGroupsIterator.hasNext()) {
            int driverGroupId = driverGroupsIterator.nextInt();
            scheduler.startLifespan(Lifespan.driverGroup(driverGroupId), partitionHandles.get(driverGroupId));
            totalDriverGroupsScheduled++;
            driverGroupsScheduled++;
            if (concurrentLifespansPerTask.isPresent() && driverGroupsScheduled == concurrentLifespansPerTask.getAsInt()) {
                break;
            }
        }
    }
    verify(totalDriverGroupsScheduled <= driverGroupToNodeMap.size());
    if (totalDriverGroupsScheduled == driverGroupToNodeMap.size()) {
        scheduler.noMoreLifespans();
    }
}

Example 25 with InternalNode

use of io.prestosql.metadata.InternalNode in project hetu-core by openlookeng.

the class DynamicCatalogStore method isAtLeastOneNodeAdded.

private boolean isAtLeastOneNodeAdded(String catalogName) {
    refreshConnectorNodes();
    Set<InternalNode> nodes = nodeManager.getActiveConnectorNodes(new CatalogName(catalogName));
    return nodes.size() >= 1;
}