Search in sources :

Example 56 with Split

use of io.trino.metadata.Split in project trino by trinodb.

the class NodePartitioningManager method getSplitToBucket.

private ToIntFunction<Split> getSplitToBucket(Session session, PartitioningHandle partitioningHandle) {
    CatalogName catalogName = partitioningHandle.getConnectorId().orElseThrow(() -> new IllegalArgumentException("No connector ID for partitioning handle: " + partitioningHandle));
    ConnectorNodePartitioningProvider partitioningProvider = getPartitioningProvider(catalogName);
    ToIntFunction<ConnectorSplit> splitBucketFunction = partitioningProvider.getSplitBucketFunction(partitioningHandle.getTransactionHandle().orElseThrow(() -> new IllegalArgumentException("No transactionHandle for partitioning handle: " + partitioningHandle)), session.toConnectorSession(catalogName), partitioningHandle.getConnectorHandle());
    checkArgument(splitBucketFunction != null, "No partitioning %s", partitioningHandle);
    return split -> {
        int bucket;
        if (split.getConnectorSplit() instanceof EmptySplit) {
            bucket = split.getLifespan().isTaskWide() ? 0 : split.getLifespan().getId();
        } else {
            bucket = splitBucketFunction.applyAsInt(split.getConnectorSplit());
        }
        if (!split.getLifespan().isTaskWide()) {
            checkArgument(split.getLifespan().getId() == bucket);
        }
        return bucket;
    };
}
Also used : IntStream(java.util.stream.IntStream) BucketFunction(io.trino.spi.connector.BucketFunction) BucketNodeMap(io.trino.execution.scheduler.BucketNodeMap) Type(io.trino.spi.type.Type) ArrayList(java.util.ArrayList) ConcurrentMap(java.util.concurrent.ConcurrentMap) Inject(javax.inject.Inject) FixedBucketNodeMap(io.trino.execution.scheduler.FixedBucketNodeMap) CatalogName(io.trino.connector.CatalogName) NodeScheduler(io.trino.execution.scheduler.NodeScheduler) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ConnectorPartitionHandle(io.trino.spi.connector.ConnectorPartitionHandle) Objects.requireNonNull(java.util.Objects.requireNonNull) DynamicBucketNodeMap(io.trino.execution.scheduler.group.DynamicBucketNodeMap) BiMap(com.google.common.collect.BiMap) EmptySplit(io.trino.split.EmptySplit) PartitionFunction(io.trino.operator.PartitionFunction) ConnectorNodePartitioningProvider(io.trino.spi.connector.ConnectorNodePartitioningProvider) BlockTypeOperators(io.trino.type.BlockTypeOperators) ConnectorSplit(io.trino.spi.connector.ConnectorSplit) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ToIntFunction(java.util.function.ToIntFunction) BucketPartitionFunction(io.trino.operator.BucketPartitionFunction) ConnectorBucketNodeMap(io.trino.spi.connector.ConnectorBucketNodeMap) HashBiMap(com.google.common.collect.HashBiMap) InternalNode(io.trino.metadata.InternalNode) List(java.util.List) Stream(java.util.stream.Stream) Split(io.trino.metadata.Split) Optional(java.util.Optional) Collections(java.util.Collections) Session(io.trino.Session) ConnectorNodePartitioningProvider(io.trino.spi.connector.ConnectorNodePartitioningProvider) EmptySplit(io.trino.split.EmptySplit) CatalogName(io.trino.connector.CatalogName) ConnectorSplit(io.trino.spi.connector.ConnectorSplit)

Aggregations

Split (io.trino.metadata.Split)56 Test (org.testng.annotations.Test)32 InternalNode (io.trino.metadata.InternalNode)26 ConnectorSplit (io.trino.spi.connector.ConnectorSplit)24 PlanNodeId (io.trino.sql.planner.plan.PlanNodeId)22 LinkedHashSet (java.util.LinkedHashSet)17 ImmutableList (com.google.common.collect.ImmutableList)16 HashSet (java.util.HashSet)16 CatalogName (io.trino.connector.CatalogName)14 Page (io.trino.spi.Page)12 List (java.util.List)11 Optional (java.util.Optional)11 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)9 ImmutableSet (com.google.common.collect.ImmutableSet)9 ArrayList (java.util.ArrayList)9 Lifespan (io.trino.execution.Lifespan)8 RemoteTask (io.trino.execution.RemoteTask)8 TestingSplit (io.trino.testing.TestingSplit)8 Objects.requireNonNull (java.util.Objects.requireNonNull)8 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)6