Search in sources :

Example 1 with UNGROUPED_SCHEDULING

use of io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING in project hetu-core by openlookeng.

the class ExtractSpatialJoins method loadKdbTree.

private static KdbTree loadKdbTree(String tableName, Session session, Metadata metadata, SplitManager splitManager, PageSourceManager pageSourceManager, PlanNodeId nodeId) {
    QualifiedObjectName name = toQualifiedObjectName(tableName, session.getCatalog().get(), session.getSchema().get());
    TableHandle tableHandle = metadata.getTableHandle(session, name).orElseThrow(() -> new PrestoException(INVALID_SPATIAL_PARTITIONING, format("Table not found: %s", name)));
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    List<ColumnHandle> visibleColumnHandles = columnHandles.values().stream().filter(handle -> !metadata.getColumnMetadata(session, tableHandle, handle).isHidden()).collect(toImmutableList());
    checkSpatialPartitioningTable(visibleColumnHandles.size() == 1, "Expected single column for table %s, but found %s columns", name, columnHandles.size());
    ColumnHandle kdbTreeColumn = Iterables.getOnlyElement(visibleColumnHandles);
    Optional<KdbTree> kdbTree = Optional.empty();
    try (SplitSource splitSource = splitManager.getSplits(session, tableHandle, UNGROUPED_SCHEDULING, null, Optional.empty(), Collections.emptyMap(), ImmutableSet.of(), false, nodeId)) {
        while (!Thread.currentThread().isInterrupted()) {
            SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000));
            List<Split> splits = splitBatch.getSplits();
            for (Split split : splits) {
                try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, tableHandle, ImmutableList.of(kdbTreeColumn), Optional.empty())) {
                    do {
                        getFutureValue(pageSource.isBlocked());
                        Page page = pageSource.getNextPage();
                        if (page != null && page.getPositionCount() > 0) {
                            checkSpatialPartitioningTable(!kdbTree.isPresent(), "Expected exactly one row for table %s, but found more", name);
                            checkSpatialPartitioningTable(page.getPositionCount() == 1, "Expected exactly one row for table %s, but found %s rows", name, page.getPositionCount());
                            String kdbTreeJson = VARCHAR.getSlice(page.getBlock(0), 0).toStringUtf8();
                            try {
                                kdbTree = Optional.of(KdbTreeUtils.fromJson(kdbTreeJson));
                            } catch (IllegalArgumentException e) {
                                checkSpatialPartitioningTable(false, "Invalid JSON string for KDB tree: %s", e.getMessage());
                            }
                        }
                    } while (!pageSource.isFinished());
                } catch (IOException e) {
                    throw new UncheckedIOException(e);
                }
            }
            if (splitBatch.isLastBatch()) {
                break;
            }
        }
    }
    checkSpatialPartitioningTable(kdbTree.isPresent(), "Expected exactly one row for table %s, but got none", name);
    return kdbTree.get();
}
Also used : ConstantExpression(io.prestosql.spi.relation.ConstantExpression) SymbolsExtractor.extractUnique(io.prestosql.sql.planner.SymbolsExtractor.extractUnique) SystemSessionProperties.getSpatialPartitioningTableName(io.prestosql.SystemSessionProperties.getSpatialPartitioningTableName) INVALID_SPATIAL_PARTITIONING(io.prestosql.spi.StandardErrorCode.INVALID_SPATIAL_PARTITIONING) TypeProvider(io.prestosql.sql.planner.TypeProvider) Result(io.prestosql.sql.planner.iterative.Rule.Result) KdbTree(io.prestosql.geospatial.KdbTree) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) CallExpression(io.prestosql.spi.relation.CallExpression) TypeSignature.parseTypeSignature(io.prestosql.spi.type.TypeSignature.parseTypeSignature) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) KdbTreeUtils(io.prestosql.geospatial.KdbTreeUtils) Capture.newCapture(io.prestosql.matching.Capture.newCapture) FilterNode(io.prestosql.spi.plan.FilterNode) OperatorType(io.prestosql.spi.function.OperatorType) Map(java.util.Map) FunctionMetadata(io.prestosql.spi.function.FunctionMetadata) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) Type(io.prestosql.spi.type.Type) RowExpressionNodeInliner.replaceExpression(io.prestosql.expressions.RowExpressionNodeInliner.replaceExpression) Splitter(com.google.common.base.Splitter) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) UNGROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) ImmutableMap(com.google.common.collect.ImmutableMap) CastType(io.prestosql.metadata.CastType) ArrayType(io.prestosql.spi.type.ArrayType) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) SpatialJoinUtils.extractSupportedSpatialComparisons(io.prestosql.util.SpatialJoinUtils.extractSupportedSpatialComparisons) PlanNode(io.prestosql.spi.plan.PlanNode) ProjectNode(io.prestosql.spi.plan.ProjectNode) Metadata(io.prestosql.metadata.Metadata) String.format(java.lang.String.format) FunctionHandle(io.prestosql.spi.function.FunctionHandle) UncheckedIOException(java.io.UncheckedIOException) Captures(io.prestosql.matching.Captures) SpatialJoinNode(io.prestosql.sql.planner.plan.SpatialJoinNode) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) Capture(io.prestosql.matching.Capture) INNER(io.prestosql.spi.plan.JoinNode.Type.INNER) Optional(java.util.Optional) TypeSignature(io.prestosql.spi.type.TypeSignature) SystemSessionProperties.isSpatialJoinEnabled(io.prestosql.SystemSessionProperties.isSpatialJoinEnabled) NOT_PARTITIONED(io.prestosql.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) Iterables(com.google.common.collect.Iterables) Patterns.source(io.prestosql.sql.planner.plan.Patterns.source) Patterns.join(io.prestosql.sql.planner.plan.Patterns.join) INTEGER(io.prestosql.spi.type.IntegerType.INTEGER) Pattern(io.prestosql.matching.Pattern) Split(io.prestosql.metadata.Split) TableHandle(io.prestosql.spi.metadata.TableHandle) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) UnnestNode(io.prestosql.sql.planner.plan.UnnestNode) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) SpatialJoinUtils.extractSupportedSpatialFunctions(io.prestosql.util.SpatialJoinUtils.extractSupportedSpatialFunctions) PageSourceManager(io.prestosql.split.PageSourceManager) SplitSource(io.prestosql.split.SplitSource) SpatialJoinUtils(io.prestosql.util.SpatialJoinUtils) JoinNode(io.prestosql.spi.plan.JoinNode) Lifespan(io.prestosql.execution.Lifespan) Symbol(io.prestosql.spi.plan.Symbol) SpatialJoinUtils.getFlippedFunctionHandle(io.prestosql.util.SpatialJoinUtils.getFlippedFunctionHandle) TypeSignatureProvider.fromTypes(io.prestosql.sql.analyzer.TypeSignatureProvider.fromTypes) SplitBatch(io.prestosql.split.SplitSource.SplitBatch) Assignments(io.prestosql.spi.plan.Assignments) Rule(io.prestosql.sql.planner.iterative.Rule) Patterns.filter(io.prestosql.sql.planner.plan.Patterns.filter) Context(io.prestosql.sql.planner.iterative.Rule.Context) Page(io.prestosql.spi.Page) IOException(java.io.IOException) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) Expressions(io.prestosql.sql.relational.Expressions) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SplitManager(io.prestosql.split.SplitManager) RowExpression(io.prestosql.spi.relation.RowExpression) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) LEFT(io.prestosql.spi.plan.JoinNode.Type.LEFT) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) KdbTree(io.prestosql.geospatial.KdbTree) PrestoException(io.prestosql.spi.PrestoException) Page(io.prestosql.spi.Page) UncheckedIOException(java.io.UncheckedIOException) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) SplitBatch(io.prestosql.split.SplitSource.SplitBatch) TableHandle(io.prestosql.spi.metadata.TableHandle) SplitSource(io.prestosql.split.SplitSource) Split(io.prestosql.metadata.Split)

Aggregations

VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 Splitter (com.google.common.base.Splitter)1 Verify.verify (com.google.common.base.Verify.verify)1 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ImmutableSet (com.google.common.collect.ImmutableSet)1 Iterables (com.google.common.collect.Iterables)1 MoreFutures.getFutureValue (io.airlift.concurrent.MoreFutures.getFutureValue)1 Slices.utf8Slice (io.airlift.slice.Slices.utf8Slice)1 Session (io.prestosql.Session)1 SystemSessionProperties.getSpatialPartitioningTableName (io.prestosql.SystemSessionProperties.getSpatialPartitioningTableName)1 SystemSessionProperties.isSpatialJoinEnabled (io.prestosql.SystemSessionProperties.isSpatialJoinEnabled)1 Lifespan (io.prestosql.execution.Lifespan)1 RowExpressionNodeInliner.replaceExpression (io.prestosql.expressions.RowExpressionNodeInliner.replaceExpression)1 KdbTree (io.prestosql.geospatial.KdbTree)1 KdbTreeUtils (io.prestosql.geospatial.KdbTreeUtils)1 Capture (io.prestosql.matching.Capture)1 Capture.newCapture (io.prestosql.matching.Capture.newCapture)1