Examples with ColumnHandle io.trino.spi.connector.ColumnHandle used on opensource projects

Example 6 with ColumnHandle

use of io.trino.spi.connector.ColumnHandle in project trino by trinodb.

the class MetadataManager method getColumnHandles.

@Override
public Map<String, ColumnHandle> getColumnHandles(Session session, TableHandle tableHandle) {
    CatalogName catalogName = tableHandle.getCatalogName();
    ConnectorMetadata metadata = getMetadata(session, catalogName);
    Map<String, ColumnHandle> handles = metadata.getColumnHandles(session.toConnectorSession(catalogName), tableHandle.getConnectorHandle());
    ImmutableMap.Builder<String, ColumnHandle> map = ImmutableMap.builder();
    for (Entry<String, ColumnHandle> mapEntry : handles.entrySet()) {
        map.put(mapEntry.getKey().toLowerCase(ENGLISH), mapEntry.getValue());
    }
    return map.buildOrThrow();
}

Example 7 with ColumnHandle

use of io.trino.spi.connector.ColumnHandle in project trino by trinodb.

the class ExtractSpatialJoins method loadKdbTree.

private static KdbTree loadKdbTree(String tableName, Session session, Metadata metadata, SplitManager splitManager, PageSourceManager pageSourceManager) {
    QualifiedObjectName name = toQualifiedObjectName(tableName, session.getCatalog().get(), session.getSchema().get());
    TableHandle tableHandle = metadata.getTableHandle(session, name).orElseThrow(() -> new TrinoException(INVALID_SPATIAL_PARTITIONING, format("Table not found: %s", name)));
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    List<ColumnHandle> visibleColumnHandles = columnHandles.values().stream().filter(handle -> !metadata.getColumnMetadata(session, tableHandle, handle).isHidden()).collect(toImmutableList());
    checkSpatialPartitioningTable(visibleColumnHandles.size() == 1, "Expected single column for table %s, but found %s columns", name, columnHandles.size());
    ColumnHandle kdbTreeColumn = Iterables.getOnlyElement(visibleColumnHandles);
    Optional<KdbTree> kdbTree = Optional.empty();
    try (SplitSource splitSource = splitManager.getSplits(session, tableHandle, UNGROUPED_SCHEDULING, EMPTY, alwaysTrue())) {
        while (!Thread.currentThread().isInterrupted()) {
            SplitBatch splitBatch = getFutureValue(splitSource.getNextBatch(NOT_PARTITIONED, Lifespan.taskWide(), 1000));
            List<Split> splits = splitBatch.getSplits();
            for (Split split : splits) {
                try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, tableHandle, ImmutableList.of(kdbTreeColumn), DynamicFilter.EMPTY)) {
                    do {
                        getFutureValue(pageSource.isBlocked());
                        Page page = pageSource.getNextPage();
                        if (page != null && page.getPositionCount() > 0) {
                            checkSpatialPartitioningTable(kdbTree.isEmpty(), "Expected exactly one row for table %s, but found more", name);
                            checkSpatialPartitioningTable(page.getPositionCount() == 1, "Expected exactly one row for table %s, but found %s rows", name, page.getPositionCount());
                            String kdbTreeJson = VARCHAR.getSlice(page.getBlock(0), 0).toStringUtf8();
                            try {
                                kdbTree = Optional.of(KdbTreeUtils.fromJson(kdbTreeJson));
                            } catch (IllegalArgumentException e) {
                                checkSpatialPartitioningTable(false, "Invalid JSON string for KDB tree: %s", e.getMessage());
                            }
                        }
                    } while (!pageSource.isFinished());
                } catch (IOException e) {
                    throw new UncheckedIOException(e);
                }
            }
            if (splitBatch.isLastBatch()) {
                break;
            }
        }
    }
    checkSpatialPartitioningTable(kdbTree.isPresent(), "Expected exactly one row for table %s, but got none", name);
    return kdbTree.get();
}

Example 8 with ColumnHandle

use of io.trino.spi.connector.ColumnHandle in project trino by trinodb.

the class PageSourceManager method createPageSource.

@Override
public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List<ColumnHandle> columns, DynamicFilter dynamicFilter) {
    requireNonNull(columns, "columns is null");
    checkArgument(split.getCatalogName().equals(table.getCatalogName()), "mismatched split and table");
    CatalogName catalogName = split.getCatalogName();
    ConnectorPageSourceProvider provider = getPageSourceProvider(catalogName);
    TupleDomain<ColumnHandle> constraint = dynamicFilter.getCurrentPredicate();
    if (constraint.isNone()) {
        return new EmptyPageSource();
    }
    if (!isAllowPushdownIntoConnectors(session)) {
        dynamicFilter = DynamicFilter.EMPTY;
    }
    return provider.createPageSource(table.getTransaction(), session.toConnectorSession(catalogName), split.getConnectorSplit(), table.getConnectorHandle(), columns, dynamicFilter);
}

Example 9 with ColumnHandle

use of io.trino.spi.connector.ColumnHandle in project trino by trinodb.

the class PushPredicateIntoTableScan method pushFilterIntoTableScan.

public static Optional<PlanNode> pushFilterIntoTableScan(FilterNode filterNode, TableScanNode node, boolean pruneWithPredicateExpression, Session session, SymbolAllocator symbolAllocator, PlannerContext plannerContext, TypeAnalyzer typeAnalyzer, StatsProvider statsProvider, DomainTranslator domainTranslator) {
    if (!isAllowPushdownIntoConnectors(session)) {
        return Optional.empty();
    }
    SplitExpression splitExpression = splitExpression(plannerContext, filterNode.getPredicate());
    DomainTranslator.ExtractionResult decomposedPredicate = DomainTranslator.getExtractionResult(plannerContext, session, splitExpression.getDeterministicPredicate(), symbolAllocator.getTypes());
    TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transformKeys(node.getAssignments()::get).intersect(node.getEnforcedConstraint());
    Map<NodeRef<Expression>, Type> remainingExpressionTypes = typeAnalyzer.getTypes(session, symbolAllocator.getTypes(), decomposedPredicate.getRemainingExpression());
    Optional<ConnectorExpression> connectorExpression = new ConnectorExpressionTranslator.SqlToConnectorExpressionTranslator(session, remainingExpressionTypes, plannerContext).process(decomposedPredicate.getRemainingExpression());
    Map<String, ColumnHandle> connectorExpressionAssignments = connectorExpression.map(ignored -> node.getAssignments().entrySet().stream().collect(toImmutableMap(entry -> entry.getKey().getName(), Map.Entry::getValue))).orElse(ImmutableMap.of());
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    Constraint constraint;
    // use evaluator only when there is some predicate which could not be translated into tuple domain
    if (pruneWithPredicateExpression && !TRUE_LITERAL.equals(decomposedPredicate.getRemainingExpression())) {
        LayoutConstraintEvaluator evaluator = new LayoutConstraintEvaluator(plannerContext, typeAnalyzer, session, symbolAllocator.getTypes(), node.getAssignments(), combineConjuncts(plannerContext.getMetadata(), splitExpression.getDeterministicPredicate(), // which would be expensive to evaluate in the call to isCandidate below.
        domainTranslator.toPredicate(session, newDomain.simplify().transformKeys(assignments::get))));
        constraint = new Constraint(newDomain, connectorExpression.orElse(TRUE), connectorExpressionAssignments, evaluator::isCandidate, evaluator.getArguments());
    } else {
        // Currently, invoking the expression interpreter is very expensive.
        // TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough.
        constraint = new Constraint(newDomain, connectorExpression.orElse(TRUE), connectorExpressionAssignments);
    }
    // check if new domain is wider than domain already provided by table scan
    if (constraint.predicate().isEmpty() && // TODO do we need to track enforced ConnectorExpression in TableScanNode?
    TRUE.equals(connectorExpression.orElse(TRUE)) && newDomain.contains(node.getEnforcedConstraint())) {
        Expression resultingPredicate = createResultingPredicate(plannerContext, session, symbolAllocator, typeAnalyzer, splitExpression.getDynamicFilter(), TRUE_LITERAL, splitExpression.getNonDeterministicPredicate(), decomposedPredicate.getRemainingExpression());
        if (!TRUE_LITERAL.equals(resultingPredicate)) {
            return Optional.of(new FilterNode(filterNode.getId(), node, resultingPredicate));
        }
        return Optional.of(node);
    }
    if (newDomain.isNone()) {
        // to turn the subtree into a Values node
        return Optional.of(new ValuesNode(node.getId(), node.getOutputSymbols(), ImmutableList.of()));
    }
    Optional<ConstraintApplicationResult<TableHandle>> result = plannerContext.getMetadata().applyFilter(session, node.getTable(), constraint);
    if (result.isEmpty()) {
        return Optional.empty();
    }
    TableHandle newTable = result.get().getHandle();
    TableProperties newTableProperties = plannerContext.getMetadata().getTableProperties(session, newTable);
    Optional<TablePartitioning> newTablePartitioning = newTableProperties.getTablePartitioning();
    if (newTableProperties.getPredicate().isNone()) {
        return Optional.of(new ValuesNode(node.getId(), node.getOutputSymbols(), ImmutableList.of()));
    }
    TupleDomain<ColumnHandle> remainingFilter = result.get().getRemainingFilter();
    Optional<ConnectorExpression> remainingConnectorExpression = result.get().getRemainingExpression();
    boolean precalculateStatistics = result.get().isPrecalculateStatistics();
    verifyTablePartitioning(session, plannerContext.getMetadata(), node, newTablePartitioning);
    TableScanNode tableScan = new TableScanNode(node.getId(), newTable, node.getOutputSymbols(), node.getAssignments(), computeEnforced(newDomain, remainingFilter), // TODO (https://github.com/trinodb/trino/issues/8144) distinguish between predicate pushed down and remaining
    deriveTableStatisticsForPushdown(statsProvider, session, precalculateStatistics, filterNode), node.isUpdateTarget(), node.getUseConnectorNodePartitioning());
    Expression remainingDecomposedPredicate;
    if (remainingConnectorExpression.isEmpty() || remainingConnectorExpression.equals(connectorExpression)) {
        remainingDecomposedPredicate = decomposedPredicate.getRemainingExpression();
    } else {
        Map<String, Symbol> variableMappings = assignments.values().stream().collect(toImmutableMap(Symbol::getName, Function.identity()));
        Expression translatedExpression = ConnectorExpressionTranslator.translate(session, remainingConnectorExpression.get(), plannerContext, variableMappings, new LiteralEncoder(plannerContext));
        if (connectorExpression.isEmpty()) {
            remainingDecomposedPredicate = ExpressionUtils.combineConjuncts(plannerContext.getMetadata(), translatedExpression, decomposedPredicate.getRemainingExpression());
        } else {
            remainingDecomposedPredicate = translatedExpression;
        }
    }
    Expression resultingPredicate = createResultingPredicate(plannerContext, session, symbolAllocator, typeAnalyzer, splitExpression.getDynamicFilter(), domainTranslator.toPredicate(session, remainingFilter.transformKeys(assignments::get)), splitExpression.getNonDeterministicPredicate(), remainingDecomposedPredicate);
    if (!TRUE_LITERAL.equals(resultingPredicate)) {
        return Optional.of(new FilterNode(filterNode.getId(), tableScan, resultingPredicate));
    }
    return Optional.of(tableScan);
}

Example 10 with ColumnHandle

use of io.trino.spi.connector.ColumnHandle in project trino by trinodb.

the class PushPredicateIntoTableScan method computeEnforced.

public static TupleDomain<ColumnHandle> computeEnforced(TupleDomain<ColumnHandle> predicate, TupleDomain<ColumnHandle> unenforced) {
    // The engine requested the connector to apply a filter with a non-none TupleDomain.
    // A TupleDomain is effectively a list of column-Domain pairs.
    // The connector is expected enforce the respective domain entirely on none, some, or all of the columns.
    // 1. When the connector could enforce none of the domains, the unenforced would be equal to predicate;
    // 2. When the connector could enforce some of the domains, the unenforced would contain a subset of the column-Domain pairs;
    // 3. When the connector could enforce all of the domains, the unenforced would be TupleDomain.all().
    // In all 3 cases shown above, the unenforced is not TupleDomain.none().
    checkArgument(!unenforced.isNone());
    Map<ColumnHandle, Domain> predicateDomains = predicate.getDomains().get();
    Map<ColumnHandle, Domain> unenforcedDomains = unenforced.getDomains().get();
    ImmutableMap.Builder<ColumnHandle, Domain> enforcedDomainsBuilder = ImmutableMap.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : predicateDomains.entrySet()) {
        ColumnHandle predicateColumnHandle = entry.getKey();
        if (unenforcedDomains.containsKey(predicateColumnHandle)) {
            checkArgument(entry.getValue().equals(unenforcedDomains.get(predicateColumnHandle)), "Enforced tuple domain cannot be determined. The connector is expected to enforce the respective domain entirely on none, some, or all of the column.");
        } else {
            enforcedDomainsBuilder.put(predicateColumnHandle, entry.getValue());
        }
    }
    Map<ColumnHandle, Domain> enforcedDomains = enforcedDomainsBuilder.buildOrThrow();
    checkArgument(enforcedDomains.size() + unenforcedDomains.size() == predicateDomains.size(), "Enforced tuple domain cannot be determined. Connector returned an unenforced TupleDomain that contains columns not in predicate.");
    return TupleDomain.withColumnDomains(enforcedDomains);
}