Search in sources :

Example 1 with INVALID_SPATIAL_PARTITIONING

use of com.facebook.presto.spi.StandardErrorCode.INVALID_SPATIAL_PARTITIONING in project presto by prestodb.

the class ExtractSpatialJoins method loadKdbTree.

private static KdbTree loadKdbTree(String tableName, Session session, Metadata metadata, SplitManager splitManager, PageSourceManager pageSourceManager) {
    QualifiedObjectName name = toQualifiedObjectName(tableName, session.getCatalog().get(), session.getSchema().get());
    TableHandle tableHandle = metadata.getTableHandle(session, name).orElseThrow(() -> new PrestoException(INVALID_SPATIAL_PARTITIONING, format("Table not found: %s", name)));
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    List<ColumnHandle> visibleColumnHandles = columnHandles.values().stream().filter(handle -> !metadata.getColumnMetadata(session, tableHandle, handle).isHidden()).collect(toImmutableList());
    checkSpatialPartitioningTable(visibleColumnHandles.size() == 1, "Expected single column for table %s, but found %s columns", name, columnHandles.size());
    ColumnHandle kdbTreeColumn = Iterables.getOnlyElement(visibleColumnHandles);
    TableLayoutResult layout = metadata.getLayout(session, tableHandle, Constraint.alwaysTrue(), Optional.of(ImmutableSet.of(kdbTreeColumn)));
    TableHandle newTableHandle = layout.getLayout().getNewTableHandle();
    Optional<KdbTree> kdbTree = Optional.empty();
    try (SplitSource splitSource = splitManager.getSplits(session, newTableHandle, UNGROUPED_SCHEDULING, WarningCollector.NOOP)) {
        while (!Thread.currentThread().isInterrupted()) {
            SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000));
            List<Split> splits = splitBatch.getSplits();
            for (Split split : splits) {
                try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, newTableHandle, ImmutableList.of(kdbTreeColumn))) {
                    do {
                        getFutureValue(pageSource.isBlocked());
                        Page page = pageSource.getNextPage();
                        if (page != null && page.getPositionCount() > 0) {
                            checkSpatialPartitioningTable(!kdbTree.isPresent(), "Expected exactly one row for table %s, but found more", name);
                            checkSpatialPartitioningTable(page.getPositionCount() == 1, "Expected exactly one row for table %s, but found %s rows", name, page.getPositionCount());
                            String kdbTreeJson = VARCHAR.getSlice(page.getBlock(0), 0).toStringUtf8();
                            try {
                                kdbTree = Optional.of(KdbTreeUtils.fromJson(kdbTreeJson));
                            } catch (IllegalArgumentException e) {
                                checkSpatialPartitioningTable(false, "Invalid JSON string for KDB tree: %s", e.getMessage());
                            }
                        }
                    } while (!pageSource.isFinished());
                } catch (IOException e) {
                    throw new UncheckedIOException(e);
                }
            }
            if (splitBatch.isLastBatch()) {
                break;
            }
        }
    }
    checkSpatialPartitioningTable(kdbTree.isPresent(), "Expected exactly one row for table %s, but got none", name);
    return kdbTree.get();
}
Also used : FunctionAndTypeManager(com.facebook.presto.metadata.FunctionAndTypeManager) WarningCollector(com.facebook.presto.spi.WarningCollector) Page(com.facebook.presto.common.Page) SpatialJoinNode(com.facebook.presto.sql.planner.plan.SpatialJoinNode) VariableReferenceExpression(com.facebook.presto.spi.relation.VariableReferenceExpression) TypeSignature(com.facebook.presto.common.type.TypeSignature) MoreFutures.getFutureValue(com.facebook.airlift.concurrent.MoreFutures.getFutureValue) NOT_PARTITIONED(com.facebook.presto.spi.connector.NotPartitionedPartitionHandle.NOT_PARTITIONED) SpatialJoinUtils.getFlippedFunctionHandle(com.facebook.presto.util.SpatialJoinUtils.getFlippedFunctionHandle) Pattern(com.facebook.presto.matching.Pattern) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Capture(com.facebook.presto.matching.Capture) SpatialJoinUtils.flip(com.facebook.presto.util.SpatialJoinUtils.flip) Map(java.util.Map) UNGROUPED_SCHEDULING(com.facebook.presto.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) LOCAL(com.facebook.presto.spi.plan.ProjectNode.Locality.LOCAL) QualifiedObjectName(com.facebook.presto.common.QualifiedObjectName) SystemSessionProperties.isSpatialJoinEnabled(com.facebook.presto.SystemSessionProperties.isSpatialJoinEnabled) Slices.utf8Slice(io.airlift.slice.Slices.utf8Slice) CallExpression(com.facebook.presto.spi.relation.CallExpression) Splitter(com.google.common.base.Splitter) SplitSource(com.facebook.presto.split.SplitSource) Lifespan(com.facebook.presto.execution.Lifespan) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) SplitManager(com.facebook.presto.split.SplitManager) String.format(java.lang.String.format) KDB_TREE(com.facebook.presto.common.type.KdbTreeType.KDB_TREE) UncheckedIOException(java.io.UncheckedIOException) FunctionMetadata(com.facebook.presto.spi.function.FunctionMetadata) List(java.util.List) SpatialJoinUtils.extractSupportedSpatialFunctions(com.facebook.presto.util.SpatialJoinUtils.extractSupportedSpatialFunctions) KdbTreeUtils(com.facebook.presto.geospatial.KdbTreeUtils) ProjectNode(com.facebook.presto.spi.plan.ProjectNode) Capture.newCapture(com.facebook.presto.matching.Capture.newCapture) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) Optional(java.util.Optional) CAST(com.facebook.presto.metadata.CastType.CAST) VariablesExtractor.extractUnique(com.facebook.presto.sql.planner.VariablesExtractor.extractUnique) INNER(com.facebook.presto.sql.planner.plan.JoinNode.Type.INNER) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) Iterables(com.google.common.collect.Iterables) TableLayoutResult(com.facebook.presto.metadata.TableLayoutResult) VARCHAR(com.facebook.presto.common.type.VarcharType.VARCHAR) Captures(com.facebook.presto.matching.Captures) Assignments(com.facebook.presto.spi.plan.Assignments) Result(com.facebook.presto.sql.planner.iterative.Rule.Result) PrestoException(com.facebook.presto.spi.PrestoException) Patterns.join(com.facebook.presto.sql.planner.plan.Patterns.join) TypeSignatureProvider.fromTypes(com.facebook.presto.sql.analyzer.TypeSignatureProvider.fromTypes) Patterns.filter(com.facebook.presto.sql.planner.plan.Patterns.filter) FilterNode(com.facebook.presto.spi.plan.FilterNode) SplitBatch(com.facebook.presto.split.SplitSource.SplitBatch) RowExpressionNodeInliner(com.facebook.presto.expressions.RowExpressionNodeInliner) ImmutableList(com.google.common.collect.ImmutableList) PageSourceManager(com.facebook.presto.split.PageSourceManager) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) SpatialJoinUtils.extractSupportedSpatialComparisons(com.facebook.presto.util.SpatialJoinUtils.extractSupportedSpatialComparisons) ArrayType(com.facebook.presto.common.type.ArrayType) TableHandle(com.facebook.presto.spi.TableHandle) Expressions(com.facebook.presto.sql.relational.Expressions) KdbTree(com.facebook.presto.geospatial.KdbTree) UnnestNode(com.facebook.presto.sql.planner.plan.UnnestNode) Type(com.facebook.presto.common.type.Type) RowExpression(com.facebook.presto.spi.relation.RowExpression) JoinNode(com.facebook.presto.sql.planner.plan.JoinNode) INVALID_SPATIAL_PARTITIONING(com.facebook.presto.spi.StandardErrorCode.INVALID_SPATIAL_PARTITIONING) SystemSessionProperties.getSpatialPartitioningTableName(com.facebook.presto.SystemSessionProperties.getSpatialPartitioningTableName) Session(com.facebook.presto.Session) Rule(com.facebook.presto.sql.planner.iterative.Rule) Constraint(com.facebook.presto.spi.Constraint) IOException(java.io.IOException) OperatorType(com.facebook.presto.common.function.OperatorType) Patterns.source(com.facebook.presto.sql.planner.plan.Patterns.source) PlanNode(com.facebook.presto.spi.plan.PlanNode) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TypeSignature.parseTypeSignature(com.facebook.presto.common.type.TypeSignature.parseTypeSignature) LEFT(com.facebook.presto.sql.planner.plan.JoinNode.Type.LEFT) ColumnHandle(com.facebook.presto.spi.ColumnHandle) FunctionHandle(com.facebook.presto.spi.function.FunctionHandle) Split(com.facebook.presto.metadata.Split) Context(com.facebook.presto.sql.planner.iterative.Rule.Context) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Metadata(com.facebook.presto.metadata.Metadata) RowExpressionNodeInliner.replaceExpression(com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression) ColumnHandle(com.facebook.presto.spi.ColumnHandle) KdbTree(com.facebook.presto.geospatial.KdbTree) PrestoException(com.facebook.presto.spi.PrestoException) Page(com.facebook.presto.common.Page) UncheckedIOException(java.io.UncheckedIOException) UncheckedIOException(java.io.UncheckedIOException) IOException(java.io.IOException) TableLayoutResult(com.facebook.presto.metadata.TableLayoutResult) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) QualifiedObjectName(com.facebook.presto.common.QualifiedObjectName) SplitBatch(com.facebook.presto.split.SplitSource.SplitBatch) TableHandle(com.facebook.presto.spi.TableHandle) SplitSource(com.facebook.presto.split.SplitSource) Split(com.facebook.presto.metadata.Split)

Aggregations

MoreFutures.getFutureValue (com.facebook.airlift.concurrent.MoreFutures.getFutureValue)1 Session (com.facebook.presto.Session)1 SystemSessionProperties.getSpatialPartitioningTableName (com.facebook.presto.SystemSessionProperties.getSpatialPartitioningTableName)1 SystemSessionProperties.isSpatialJoinEnabled (com.facebook.presto.SystemSessionProperties.isSpatialJoinEnabled)1 Page (com.facebook.presto.common.Page)1 QualifiedObjectName (com.facebook.presto.common.QualifiedObjectName)1 OperatorType (com.facebook.presto.common.function.OperatorType)1 ArrayType (com.facebook.presto.common.type.ArrayType)1 INTEGER (com.facebook.presto.common.type.IntegerType.INTEGER)1 KDB_TREE (com.facebook.presto.common.type.KdbTreeType.KDB_TREE)1 Type (com.facebook.presto.common.type.Type)1 TypeSignature (com.facebook.presto.common.type.TypeSignature)1 TypeSignature.parseTypeSignature (com.facebook.presto.common.type.TypeSignature.parseTypeSignature)1 VARCHAR (com.facebook.presto.common.type.VarcharType.VARCHAR)1 Lifespan (com.facebook.presto.execution.Lifespan)1 RowExpressionNodeInliner (com.facebook.presto.expressions.RowExpressionNodeInliner)1 RowExpressionNodeInliner.replaceExpression (com.facebook.presto.expressions.RowExpressionNodeInliner.replaceExpression)1 KdbTree (com.facebook.presto.geospatial.KdbTree)1 KdbTreeUtils (com.facebook.presto.geospatial.KdbTreeUtils)1 Capture (com.facebook.presto.matching.Capture)1