Search in sources :

Example 1 with NodeMap

use of com.facebook.presto.execution.scheduler.NodeMap in project presto by prestodb.

the class SimpleNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks) {
    Multimap<InternalNode, Split> assignment = HashMultimap.create();
    NodeMap nodeMap = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    List<InternalNode> eligibleNodes = getEligibleNodes(maxTasksPerStage, nodeMap, existingTasks);
    NodeSelection randomNodeSelection = new RandomNodeSelection(eligibleNodes, minCandidates);
    Set<InternalNode> blockedExactNodes = new HashSet<>();
    boolean splitWaitingForAnyNode = false;
    NodeProvider nodeProvider = nodeMap.getActiveNodeProvider(nodeSelectionHashStrategy);
    OptionalInt preferredNodeCount = OptionalInt.empty();
    for (Split split : splits) {
        List<InternalNode> candidateNodes;
        switch(split.getNodeSelectionStrategy()) {
            case HARD_AFFINITY:
                candidateNodes = selectExactNodes(nodeMap, split.getPreferredNodes(nodeProvider), includeCoordinator);
                preferredNodeCount = OptionalInt.of(candidateNodes.size());
                break;
            case SOFT_AFFINITY:
                // Using all nodes for soft affinity scheduling with modular hashing because otherwise temporarily down nodes would trigger too much rehashing
                if (nodeSelectionHashStrategy == MODULAR_HASHING) {
                    nodeProvider = new ModularHashingNodeProvider(nodeMap.getAllNodes());
                }
                candidateNodes = selectExactNodes(nodeMap, split.getPreferredNodes(nodeProvider), includeCoordinator);
                preferredNodeCount = OptionalInt.of(candidateNodes.size());
                candidateNodes = ImmutableList.<InternalNode>builder().addAll(candidateNodes).addAll(randomNodeSelection.pickNodes(split)).build();
                break;
            case NO_PREFERENCE:
                candidateNodes = randomNodeSelection.pickNodes(split);
                break;
            default:
                throw new PrestoException(NODE_SELECTION_NOT_SUPPORTED, format("Unsupported node selection strategy %s", split.getNodeSelectionStrategy()));
        }
        if (candidateNodes.isEmpty()) {
            log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getActiveNodes());
            throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
        }
        SplitWeight splitWeight = split.getSplitWeight();
        Optional<InternalNodeInfo> chosenNodeInfo = chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getTotalSplitsWeight, preferredNodeCount, maxSplitsWeightPerNode, assignmentStats);
        if (!chosenNodeInfo.isPresent()) {
            chosenNodeInfo = chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getQueuedSplitsWeightForStage, preferredNodeCount, maxPendingSplitsWeightPerTask, assignmentStats);
        }
        if (chosenNodeInfo.isPresent()) {
            split = new Split(split.getConnectorId(), split.getTransactionHandle(), split.getConnectorSplit(), split.getLifespan(), new SplitContext(chosenNodeInfo.get().isCacheable()));
            InternalNode chosenNode = chosenNodeInfo.get().getInternalNode();
            assignment.put(chosenNode, split);
            assignmentStats.addAssignedSplit(chosenNode, splitWeight);
        } else {
            if (split.getNodeSelectionStrategy() != HARD_AFFINITY) {
                splitWaitingForAnyNode = true;
            } else // Exact node set won't matter, if a split is waiting for any node
            if (!splitWaitingForAnyNode) {
                blockedExactNodes.addAll(candidateNodes);
            }
        }
    }
    ListenableFuture<?> blocked;
    if (splitWaitingForAnyNode) {
        blocked = toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingSplitsWeightPerTask));
    } else {
        blocked = toWhenHasSplitQueueSpaceFuture(blockedExactNodes, existingTasks, calculateLowWatermark(maxPendingSplitsWeightPerTask));
    }
    return new SplitPlacementResult(blocked, assignment);
}
Also used : NodeAssignmentStats(com.facebook.presto.execution.scheduler.NodeAssignmentStats) InternalNodeInfo(com.facebook.presto.execution.scheduler.InternalNodeInfo) PrestoException(com.facebook.presto.spi.PrestoException) NodeProvider(com.facebook.presto.spi.NodeProvider) ModularHashingNodeProvider(com.facebook.presto.execution.scheduler.ModularHashingNodeProvider) OptionalInt(java.util.OptionalInt) ModularHashingNodeProvider(com.facebook.presto.execution.scheduler.ModularHashingNodeProvider) SplitWeight(com.facebook.presto.spi.SplitWeight) SplitContext(com.facebook.presto.spi.SplitContext) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) NodeMap(com.facebook.presto.execution.scheduler.NodeMap) InternalNode(com.facebook.presto.metadata.InternalNode) Split(com.facebook.presto.metadata.Split) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) Sets.newHashSet(com.google.common.collect.Sets.newHashSet) HashSet(java.util.HashSet)

Example 2 with NodeMap

use of com.facebook.presto.execution.scheduler.NodeMap in project presto by prestodb.

the class SimpleTtlNodeSelector method getEligibleNodes.

private List<InternalNode> getEligibleNodes(int limit, NodeMap nodeMap, List<RemoteTask> existingTasks) {
    Map<InternalNode, NodeTtl> nodeTtlInfo = nodeTtlFetcherManager.getAllTtls();
    Map<InternalNode, Optional<ConfidenceBasedTtlInfo>> ttlInfo = nodeTtlInfo.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue().getTtlInfo().stream().min(Comparator.comparing(ConfidenceBasedTtlInfo::getExpiryInstant))));
    Duration estimatedExecutionTimeRemaining = getEstimatedExecutionTimeRemaining();
    // Of the nodes on which already have existing tasks, pick only those whose TTL is enough
    List<InternalNode> existingEligibleNodes = existingTasks.stream().map(remoteTask -> nodeMap.getActiveNodesByNodeId().get(remoteTask.getNodeId())).filter(Objects::nonNull).filter(ttlInfo::containsKey).filter(node -> ttlInfo.get(node).isPresent()).filter(node -> isTtlEnough(ttlInfo.get(node).get(), estimatedExecutionTimeRemaining)).collect(toList());
    int alreadySelectedNodeCount = existingEligibleNodes.size();
    List<InternalNode> activeNodes = nodeMap.getActiveNodes();
    List<InternalNode> newEligibleNodes = filterNodesByTtl(activeNodes, ImmutableSet.copyOf(existingEligibleNodes), ttlInfo, estimatedExecutionTimeRemaining);
    if (alreadySelectedNodeCount < limit && newEligibleNodes.size() > 0) {
        List<InternalNode> moreNodes = selectNodes(limit - alreadySelectedNodeCount, new ResettableRandomizedIterator<>(newEligibleNodes));
        existingEligibleNodes.addAll(moreNodes);
    }
    verify(existingEligibleNodes.stream().allMatch(Objects::nonNull), "existingNodes list must not contain any nulls");
    return existingEligibleNodes;
}
Also used : NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) NodeScheduler.selectNodes(com.facebook.presto.execution.scheduler.NodeScheduler.selectNodes) Duration(io.airlift.units.Duration) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SplitContext(com.facebook.presto.spi.SplitContext) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) Map(java.util.Map) ConfidenceBasedTtlInfo(com.facebook.presto.spi.ttl.ConfidenceBasedTtlInfo) NODE_SELECTION_NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NODE_SELECTION_NOT_SUPPORTED) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) NO_NODES_AVAILABLE(com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE) Instant(java.time.Instant) String.format(java.lang.String.format) Objects(java.util.Objects) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) NodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManager) ResettableRandomizedIterator(com.facebook.presto.execution.scheduler.ResettableRandomizedIterator) NodeTtl(com.facebook.presto.spi.ttl.NodeTtl) Optional(java.util.Optional) NodeScheduler.calculateLowWatermark(com.facebook.presto.execution.scheduler.NodeScheduler.calculateLowWatermark) Logger(com.facebook.airlift.log.Logger) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) InternalNodeInfo(com.facebook.presto.execution.scheduler.InternalNodeInfo) Supplier(com.google.common.base.Supplier) NodeMap(com.facebook.presto.execution.scheduler.NodeMap) QueryManager(com.facebook.presto.execution.QueryManager) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) AtomicReference(java.util.concurrent.atomic.AtomicReference) NodeScheduler.toWhenHasSplitQueueSpaceFuture(com.facebook.presto.execution.scheduler.NodeScheduler.toWhenHasSplitQueueSpaceFuture) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) Suppliers(com.google.common.base.Suppliers) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) SECONDS(java.time.temporal.ChronoUnit.SECONDS) Futures.immediateFuture(com.google.common.util.concurrent.Futures.immediateFuture) Session(com.facebook.presto.Session) NodeSelectionStrategy(com.facebook.presto.spi.schedule.NodeSelectionStrategy) NodeAssignmentStats(com.facebook.presto.execution.scheduler.NodeAssignmentStats) InternalNode(com.facebook.presto.metadata.InternalNode) TimeUnit(java.util.concurrent.TimeUnit) Collectors.toList(java.util.stream.Collectors.toList) RemoteTask(com.facebook.presto.execution.RemoteTask) Split(com.facebook.presto.metadata.Split) VisibleForTesting(com.google.common.annotations.VisibleForTesting) SplitWeight(com.facebook.presto.spi.SplitWeight) Comparator(java.util.Comparator) Optional(java.util.Optional) Objects(java.util.Objects) NodeTtl(com.facebook.presto.spi.ttl.NodeTtl) Duration(io.airlift.units.Duration) InternalNode(com.facebook.presto.metadata.InternalNode) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) NodeMap(com.facebook.presto.execution.scheduler.NodeMap)

Example 3 with NodeMap

use of com.facebook.presto.execution.scheduler.NodeMap in project presto by prestodb.

the class SimpleTtlNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks) {
    boolean isNodeSelectionStrategyNoPreference = splits.stream().allMatch(split -> split.getNodeSelectionStrategy() == NodeSelectionStrategy.NO_PREFERENCE);
    // Current NodeSelectionStrategy support is limited to NO_PREFERENCE
    if (!isNodeSelectionStrategyNoPreference) {
        return simpleNodeSelector.computeAssignments(splits, existingTasks);
    }
    ImmutableMultimap.Builder<InternalNode, Split> assignment = ImmutableMultimap.builder();
    NodeMap nodeMap = this.nodeMap.get().get();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    List<InternalNode> eligibleNodes = getEligibleNodes(maxTasksPerStage, nodeMap, existingTasks);
    NodeSelection randomNodeSelection = new RandomNodeSelection(eligibleNodes, minCandidates);
    boolean splitWaitingForAnyNode = false;
    OptionalInt preferredNodeCount = OptionalInt.empty();
    for (Split split : splits) {
        if (split.getNodeSelectionStrategy() != NodeSelectionStrategy.NO_PREFERENCE) {
            throw new PrestoException(NODE_SELECTION_NOT_SUPPORTED, format("Unsupported node selection strategy for TTL scheduling: %s", split.getNodeSelectionStrategy()));
        }
        List<InternalNode> candidateNodes = randomNodeSelection.pickNodes(split);
        if (candidateNodes.isEmpty()) {
            log.warn("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getActiveNodes());
            throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
        }
        SplitWeight splitWeight = split.getSplitWeight();
        Optional<InternalNodeInfo> chosenNodeInfo = simpleNodeSelector.chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getTotalSplitsWeight, preferredNodeCount, maxSplitsWeightPerNode, assignmentStats);
        if (!chosenNodeInfo.isPresent()) {
            chosenNodeInfo = simpleNodeSelector.chooseLeastBusyNode(splitWeight, candidateNodes, assignmentStats::getQueuedSplitsWeightForStage, preferredNodeCount, maxPendingSplitsWeightPerTask, assignmentStats);
        }
        if (chosenNodeInfo.isPresent()) {
            split = new Split(split.getConnectorId(), split.getTransactionHandle(), split.getConnectorSplit(), split.getLifespan(), new SplitContext(chosenNodeInfo.get().isCacheable()));
            InternalNode chosenNode = chosenNodeInfo.get().getInternalNode();
            assignment.put(chosenNode, split);
            assignmentStats.addAssignedSplit(chosenNode, splitWeight);
        } else {
            splitWaitingForAnyNode = true;
        }
    }
    ListenableFuture<?> blocked = splitWaitingForAnyNode ? toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingSplitsWeightPerTask)) : immediateFuture(null);
    return new SplitPlacementResult(blocked, assignment.build());
}
Also used : NodeAssignmentStats(com.facebook.presto.execution.scheduler.NodeAssignmentStats) InternalNodeInfo(com.facebook.presto.execution.scheduler.InternalNodeInfo) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) SplitWeight(com.facebook.presto.spi.SplitWeight) SplitContext(com.facebook.presto.spi.SplitContext) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) NodeMap(com.facebook.presto.execution.scheduler.NodeMap) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) InternalNode(com.facebook.presto.metadata.InternalNode) Split(com.facebook.presto.metadata.Split) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult)

Example 4 with NodeMap

use of com.facebook.presto.execution.scheduler.NodeMap in project presto by prestodb.

the class SimpleTtlNodeSelector method selectRandomNodes.

@Override
public List<InternalNode> selectRandomNodes(int limit, Set<InternalNode> excludedNodes) {
    Map<InternalNode, NodeTtl> nodeTtlInfo = nodeTtlFetcherManager.getAllTtls();
    Map<InternalNode, Optional<ConfidenceBasedTtlInfo>> ttlInfo = nodeTtlInfo.entrySet().stream().collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue().getTtlInfo().stream().min(Comparator.comparing(ConfidenceBasedTtlInfo::getExpiryInstant))));
    NodeMap nodeMap = this.nodeMap.get().get();
    List<InternalNode> activeNodes = nodeMap.getActiveNodes();
    Duration estimatedExecutionTimeRemaining = getEstimatedExecutionTimeRemaining();
    List<InternalNode> eligibleNodes = filterNodesByTtl(activeNodes, excludedNodes, ttlInfo, estimatedExecutionTimeRemaining);
    return selectNodes(limit, new ResettableRandomizedIterator<>(eligibleNodes));
}
Also used : NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) NodeScheduler.selectNodes(com.facebook.presto.execution.scheduler.NodeScheduler.selectNodes) Duration(io.airlift.units.Duration) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) SplitContext(com.facebook.presto.spi.SplitContext) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) Map(java.util.Map) ConfidenceBasedTtlInfo(com.facebook.presto.spi.ttl.ConfidenceBasedTtlInfo) NODE_SELECTION_NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NODE_SELECTION_NOT_SUPPORTED) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) NO_NODES_AVAILABLE(com.facebook.presto.spi.StandardErrorCode.NO_NODES_AVAILABLE) Instant(java.time.Instant) String.format(java.lang.String.format) Objects(java.util.Objects) List(java.util.List) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) NodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManager) ResettableRandomizedIterator(com.facebook.presto.execution.scheduler.ResettableRandomizedIterator) NodeTtl(com.facebook.presto.spi.ttl.NodeTtl) Optional(java.util.Optional) NodeScheduler.calculateLowWatermark(com.facebook.presto.execution.scheduler.NodeScheduler.calculateLowWatermark) Logger(com.facebook.airlift.log.Logger) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) InternalNodeInfo(com.facebook.presto.execution.scheduler.InternalNodeInfo) Supplier(com.google.common.base.Supplier) NodeMap(com.facebook.presto.execution.scheduler.NodeMap) QueryManager(com.facebook.presto.execution.QueryManager) PrestoException(com.facebook.presto.spi.PrestoException) OptionalInt(java.util.OptionalInt) AtomicReference(java.util.concurrent.atomic.AtomicReference) NodeScheduler.toWhenHasSplitQueueSpaceFuture(com.facebook.presto.execution.scheduler.NodeScheduler.toWhenHasSplitQueueSpaceFuture) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) Suppliers(com.google.common.base.Suppliers) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) SECONDS(java.time.temporal.ChronoUnit.SECONDS) Futures.immediateFuture(com.google.common.util.concurrent.Futures.immediateFuture) Session(com.facebook.presto.Session) NodeSelectionStrategy(com.facebook.presto.spi.schedule.NodeSelectionStrategy) NodeAssignmentStats(com.facebook.presto.execution.scheduler.NodeAssignmentStats) InternalNode(com.facebook.presto.metadata.InternalNode) TimeUnit(java.util.concurrent.TimeUnit) Collectors.toList(java.util.stream.Collectors.toList) RemoteTask(com.facebook.presto.execution.RemoteTask) Split(com.facebook.presto.metadata.Split) VisibleForTesting(com.google.common.annotations.VisibleForTesting) SplitWeight(com.facebook.presto.spi.SplitWeight) Comparator(java.util.Comparator) Optional(java.util.Optional) NodeTtl(com.facebook.presto.spi.ttl.NodeTtl) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) NodeMap(com.facebook.presto.execution.scheduler.NodeMap) Duration(io.airlift.units.Duration) InternalNode(com.facebook.presto.metadata.InternalNode) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) Map(java.util.Map) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) NodeMap(com.facebook.presto.execution.scheduler.NodeMap)

Example 5 with NodeMap

use of com.facebook.presto.execution.scheduler.NodeMap in project presto by prestodb.

the class TopologyAwareNodeSelector method computeAssignments.

@Override
public SplitPlacementResult computeAssignments(Set<Split> splits, List<RemoteTask> existingTasks) {
    NodeMap nodeMap = this.nodeMap.get().get();
    Multimap<InternalNode, Split> assignment = HashMultimap.create();
    NodeAssignmentStats assignmentStats = new NodeAssignmentStats(nodeTaskMap, nodeMap, existingTasks);
    int[] topologicCounters = new int[topologicalSplitCounters.size()];
    Set<NetworkLocation> filledLocations = new HashSet<>();
    Set<InternalNode> blockedExactNodes = new HashSet<>();
    boolean splitWaitingForAnyNode = false;
    NodeProvider nodeProvider = nodeMap.getActiveNodeProvider(nodeSelectionHashStrategy);
    for (Split split : splits) {
        SplitWeight splitWeight = split.getSplitWeight();
        if (split.getNodeSelectionStrategy() == HARD_AFFINITY) {
            List<InternalNode> candidateNodes = selectExactNodes(nodeMap, split.getPreferredNodes(nodeProvider), includeCoordinator);
            if (candidateNodes.isEmpty()) {
                log.debug("No nodes available to schedule %s. Available nodes %s", split, nodeMap.getActiveNodes());
                throw new PrestoException(NO_NODES_AVAILABLE, "No nodes available to run query");
            }
            InternalNode chosenNode = bestNodeSplitCount(splitWeight, candidateNodes.iterator(), minCandidates, maxPendingSplitsWeightPerTask, assignmentStats);
            if (chosenNode != null) {
                assignment.put(chosenNode, split);
                assignmentStats.addAssignedSplit(chosenNode, splitWeight);
            } else // Exact node set won't matter, if a split is waiting for any node
            if (!splitWaitingForAnyNode) {
                blockedExactNodes.addAll(candidateNodes);
            }
            continue;
        }
        InternalNode chosenNode = null;
        int depth = networkLocationSegmentNames.size();
        int chosenDepth = 0;
        Set<NetworkLocation> locations = new HashSet<>();
        for (HostAddress host : split.getPreferredNodes(nodeProvider)) {
            locations.add(networkLocationCache.get(host));
        }
        if (locations.isEmpty()) {
            // Add the root location
            locations.add(ROOT_LOCATION);
            depth = 0;
        }
        // Try each address at progressively shallower network locations
        for (int i = depth; i >= 0 && chosenNode == null; i--) {
            for (NetworkLocation location : locations) {
                // For example, locations which couldn't be located will be at the "root" location
                if (location.getSegments().size() < i) {
                    continue;
                }
                location = location.subLocation(0, i);
                if (filledLocations.contains(location)) {
                    continue;
                }
                Set<InternalNode> nodes = nodeMap.getActiveWorkersByNetworkPath().get(location);
                chosenNode = bestNodeSplitCount(splitWeight, new ResettableRandomizedIterator<>(nodes), minCandidates, calculateMaxPendingSplitsWeightPerTask(i, depth), assignmentStats);
                if (chosenNode != null) {
                    chosenDepth = i;
                    break;
                }
                filledLocations.add(location);
            }
        }
        if (chosenNode != null) {
            assignment.put(chosenNode, split);
            assignmentStats.addAssignedSplit(chosenNode, splitWeight);
            topologicCounters[chosenDepth]++;
        } else {
            splitWaitingForAnyNode = true;
        }
    }
    for (int i = 0; i < topologicCounters.length; i++) {
        if (topologicCounters[i] > 0) {
            topologicalSplitCounters.get(i).update(topologicCounters[i]);
        }
    }
    ListenableFuture<?> blocked;
    long maxPendingForWildcardNetworkAffinity = calculateMaxPendingSplitsWeightPerTask(0, networkLocationSegmentNames.size());
    if (splitWaitingForAnyNode) {
        blocked = toWhenHasSplitQueueSpaceFuture(existingTasks, calculateLowWatermark(maxPendingForWildcardNetworkAffinity));
    } else {
        blocked = toWhenHasSplitQueueSpaceFuture(blockedExactNodes, existingTasks, calculateLowWatermark(maxPendingForWildcardNetworkAffinity));
    }
    return new SplitPlacementResult(blocked, assignment);
}
Also used : NodeAssignmentStats(com.facebook.presto.execution.scheduler.NodeAssignmentStats) PrestoException(com.facebook.presto.spi.PrestoException) NodeProvider(com.facebook.presto.spi.NodeProvider) HostAddress(com.facebook.presto.spi.HostAddress) NetworkLocation(com.facebook.presto.execution.scheduler.NetworkLocation) SplitWeight(com.facebook.presto.spi.SplitWeight) ResettableRandomizedIterator(com.facebook.presto.execution.scheduler.ResettableRandomizedIterator) NodeMap(com.facebook.presto.execution.scheduler.NodeMap) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) InternalNode(com.facebook.presto.metadata.InternalNode) Split(com.facebook.presto.metadata.Split) SplitPlacementResult(com.facebook.presto.execution.scheduler.SplitPlacementResult) HashSet(java.util.HashSet)

Aggregations

BucketNodeMap (com.facebook.presto.execution.scheduler.BucketNodeMap)5 NodeAssignmentStats (com.facebook.presto.execution.scheduler.NodeAssignmentStats)5 NodeMap (com.facebook.presto.execution.scheduler.NodeMap)5 SplitPlacementResult (com.facebook.presto.execution.scheduler.SplitPlacementResult)5 InternalNode (com.facebook.presto.metadata.InternalNode)5 Split (com.facebook.presto.metadata.Split)5 PrestoException (com.facebook.presto.spi.PrestoException)5 SplitWeight (com.facebook.presto.spi.SplitWeight)5 InternalNodeInfo (com.facebook.presto.execution.scheduler.InternalNodeInfo)4 SplitContext (com.facebook.presto.spi.SplitContext)4 OptionalInt (java.util.OptionalInt)4 ResettableRandomizedIterator (com.facebook.presto.execution.scheduler.ResettableRandomizedIterator)3 ImmutableMultimap (com.google.common.collect.ImmutableMultimap)3 Logger (com.facebook.airlift.log.Logger)2 Session (com.facebook.presto.Session)2 NodeTaskMap (com.facebook.presto.execution.NodeTaskMap)2 QueryManager (com.facebook.presto.execution.QueryManager)2 RemoteTask (com.facebook.presto.execution.RemoteTask)2 NodeScheduler.calculateLowWatermark (com.facebook.presto.execution.scheduler.NodeScheduler.calculateLowWatermark)2 NodeScheduler.selectNodes (com.facebook.presto.execution.scheduler.NodeScheduler.selectNodes)2