Search in sources :

Example 1 with NODE_SELECTION_NOT_SUPPORTED

use of com.facebook.presto.spi.StandardErrorCode.NODE_SELECTION_NOT_SUPPORTED in project presto by prestodb.

the class NodePartitioningManager method getNodePartitioningMap.

public NodePartitionMap getNodePartitioningMap(Session session, PartitioningHandle partitioningHandle) {
    requireNonNull(session, "session is null");
    requireNonNull(partitioningHandle, "partitioningHandle is null");
    if (partitioningHandle.getConnectorHandle() instanceof SystemPartitioningHandle) {
        return ((SystemPartitioningHandle) partitioningHandle.getConnectorHandle()).getNodePartitionMap(session, nodeScheduler);
    }
    ConnectorId connectorId = partitioningHandle.getConnectorId().orElseThrow(() -> new IllegalArgumentException("No connector ID for partitioning handle: " + partitioningHandle));
    ConnectorBucketNodeMap connectorBucketNodeMap = getConnectorBucketNodeMap(session, partitioningHandle);
    // safety check for crazy partitioning
    checkArgument(connectorBucketNodeMap.getBucketCount() < 1_000_000, "Too many buckets in partitioning: %s", connectorBucketNodeMap.getBucketCount());
    List<InternalNode> bucketToNode;
    NodeSelectionStrategy nodeSelectionStrategy = connectorBucketNodeMap.getNodeSelectionStrategy();
    boolean cacheable;
    switch(nodeSelectionStrategy) {
        case HARD_AFFINITY:
            bucketToNode = getFixedMapping(connectorBucketNodeMap);
            cacheable = false;
            break;
        case SOFT_AFFINITY:
            bucketToNode = getFixedMapping(connectorBucketNodeMap);
            cacheable = true;
            break;
        case NO_PREFERENCE:
            bucketToNode = createArbitraryBucketToNode(nodeScheduler.createNodeSelector(session, connectorId).selectRandomNodes(getMaxTasksPerStage(session)), connectorBucketNodeMap.getBucketCount());
            cacheable = false;
            break;
        default:
            throw new PrestoException(NODE_SELECTION_NOT_SUPPORTED, format("Unsupported node selection strategy %s", nodeSelectionStrategy));
    }
    int[] bucketToPartition = new int[connectorBucketNodeMap.getBucketCount()];
    BiMap<InternalNode, Integer> nodeToPartition = HashBiMap.create();
    int nextPartitionId = 0;
    for (int bucket = 0; bucket < bucketToNode.size(); bucket++) {
        InternalNode node = bucketToNode.get(bucket);
        Integer partitionId = nodeToPartition.get(node);
        if (partitionId == null) {
            partitionId = nextPartitionId++;
            nodeToPartition.put(node, partitionId);
        }
        bucketToPartition[bucket] = partitionId;
    }
    List<InternalNode> partitionToNode = IntStream.range(0, nodeToPartition.size()).mapToObj(partitionId -> nodeToPartition.inverse().get(partitionId)).collect(toImmutableList());
    return new NodePartitionMap(partitionToNode, bucketToPartition, getSplitToBucket(session, partitioningHandle), cacheable);
}
Also used : IntStream(java.util.stream.IntStream) ConnectorPartitionHandle(com.facebook.presto.spi.connector.ConnectorPartitionHandle) DEAD(com.facebook.presto.metadata.InternalNode.NodeStatus.DEAD) DynamicBucketNodeMap(com.facebook.presto.execution.scheduler.group.DynamicBucketNodeMap) NodeSelectionStats(com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats) NodeScheduler(com.facebook.presto.execution.scheduler.NodeScheduler) PrestoException(com.facebook.presto.spi.PrestoException) ConnectorNodePartitioningProvider(com.facebook.presto.spi.connector.ConnectorNodePartitioningProvider) EmptySplit(com.facebook.presto.split.EmptySplit) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Node(com.facebook.presto.spi.Node) ImmutableList(com.google.common.collect.ImmutableList) BucketNodeMap(com.facebook.presto.execution.scheduler.BucketNodeMap) Objects.requireNonNull(java.util.Objects.requireNonNull) SystemSessionProperties.getMaxTasksPerStage(com.facebook.presto.SystemSessionProperties.getMaxTasksPerStage) Type(com.facebook.presto.common.type.Type) BiMap(com.google.common.collect.BiMap) BucketPartitionFunction(com.facebook.presto.operator.BucketPartitionFunction) NODE_SELECTION_NOT_SUPPORTED(com.facebook.presto.spi.StandardErrorCode.NODE_SELECTION_NOT_SUPPORTED) FixedBucketNodeMap(com.facebook.presto.execution.scheduler.FixedBucketNodeMap) Session(com.facebook.presto.Session) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ToIntFunction(java.util.function.ToIntFunction) NodeSelectionStrategy(com.facebook.presto.spi.schedule.NodeSelectionStrategy) PartitionFunction(com.facebook.presto.operator.PartitionFunction) String.format(java.lang.String.format) InternalNode(com.facebook.presto.metadata.InternalNode) ConnectorSplit(com.facebook.presto.spi.ConnectorSplit) HashBiMap(com.google.common.collect.HashBiMap) List(java.util.List) BucketFunction(com.facebook.presto.spi.BucketFunction) Split(com.facebook.presto.metadata.Split) ConnectorBucketNodeMap(com.facebook.presto.spi.connector.ConnectorBucketNodeMap) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) Collections(java.util.Collections) NodeSelectionStrategy(com.facebook.presto.spi.schedule.NodeSelectionStrategy) PrestoException(com.facebook.presto.spi.PrestoException) ConnectorBucketNodeMap(com.facebook.presto.spi.connector.ConnectorBucketNodeMap) InternalNode(com.facebook.presto.metadata.InternalNode) ConnectorId(com.facebook.presto.spi.ConnectorId)

Aggregations

Session (com.facebook.presto.Session)1 SystemSessionProperties.getMaxTasksPerStage (com.facebook.presto.SystemSessionProperties.getMaxTasksPerStage)1 Type (com.facebook.presto.common.type.Type)1 BucketNodeMap (com.facebook.presto.execution.scheduler.BucketNodeMap)1 FixedBucketNodeMap (com.facebook.presto.execution.scheduler.FixedBucketNodeMap)1 NodeScheduler (com.facebook.presto.execution.scheduler.NodeScheduler)1 DynamicBucketNodeMap (com.facebook.presto.execution.scheduler.group.DynamicBucketNodeMap)1 NodeSelectionStats (com.facebook.presto.execution.scheduler.nodeSelection.NodeSelectionStats)1 InternalNode (com.facebook.presto.metadata.InternalNode)1 DEAD (com.facebook.presto.metadata.InternalNode.NodeStatus.DEAD)1 Split (com.facebook.presto.metadata.Split)1 BucketPartitionFunction (com.facebook.presto.operator.BucketPartitionFunction)1 PartitionFunction (com.facebook.presto.operator.PartitionFunction)1 BucketFunction (com.facebook.presto.spi.BucketFunction)1 ConnectorId (com.facebook.presto.spi.ConnectorId)1 ConnectorSplit (com.facebook.presto.spi.ConnectorSplit)1 Node (com.facebook.presto.spi.Node)1 PrestoException (com.facebook.presto.spi.PrestoException)1 NODE_SELECTION_NOT_SUPPORTED (com.facebook.presto.spi.StandardErrorCode.NODE_SELECTION_NOT_SUPPORTED)1 ConnectorBucketNodeMap (com.facebook.presto.spi.connector.ConnectorBucketNodeMap)1