use of io.trino.sql.planner.DomainTranslator in project trino by trinodb.
the class PushPredicateIntoTableScan method pushFilterIntoTableScan.
public static Optional<PlanNode> pushFilterIntoTableScan(FilterNode filterNode, TableScanNode node, boolean pruneWithPredicateExpression, Session session, SymbolAllocator symbolAllocator, PlannerContext plannerContext, TypeAnalyzer typeAnalyzer, StatsProvider statsProvider, DomainTranslator domainTranslator) {
if (!isAllowPushdownIntoConnectors(session)) {
return Optional.empty();
}
SplitExpression splitExpression = splitExpression(plannerContext, filterNode.getPredicate());
DomainTranslator.ExtractionResult decomposedPredicate = DomainTranslator.getExtractionResult(plannerContext, session, splitExpression.getDeterministicPredicate(), symbolAllocator.getTypes());
TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transformKeys(node.getAssignments()::get).intersect(node.getEnforcedConstraint());
Map<NodeRef<Expression>, Type> remainingExpressionTypes = typeAnalyzer.getTypes(session, symbolAllocator.getTypes(), decomposedPredicate.getRemainingExpression());
Optional<ConnectorExpression> connectorExpression = new ConnectorExpressionTranslator.SqlToConnectorExpressionTranslator(session, remainingExpressionTypes, plannerContext).process(decomposedPredicate.getRemainingExpression());
Map<String, ColumnHandle> connectorExpressionAssignments = connectorExpression.map(ignored -> node.getAssignments().entrySet().stream().collect(toImmutableMap(entry -> entry.getKey().getName(), Map.Entry::getValue))).orElse(ImmutableMap.of());
Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
Constraint constraint;
// use evaluator only when there is some predicate which could not be translated into tuple domain
if (pruneWithPredicateExpression && !TRUE_LITERAL.equals(decomposedPredicate.getRemainingExpression())) {
LayoutConstraintEvaluator evaluator = new LayoutConstraintEvaluator(plannerContext, typeAnalyzer, session, symbolAllocator.getTypes(), node.getAssignments(), combineConjuncts(plannerContext.getMetadata(), splitExpression.getDeterministicPredicate(), // which would be expensive to evaluate in the call to isCandidate below.
domainTranslator.toPredicate(session, newDomain.simplify().transformKeys(assignments::get))));
constraint = new Constraint(newDomain, connectorExpression.orElse(TRUE), connectorExpressionAssignments, evaluator::isCandidate, evaluator.getArguments());
} else {
// Currently, invoking the expression interpreter is very expensive.
// TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough.
constraint = new Constraint(newDomain, connectorExpression.orElse(TRUE), connectorExpressionAssignments);
}
// check if new domain is wider than domain already provided by table scan
if (constraint.predicate().isEmpty() && // TODO do we need to track enforced ConnectorExpression in TableScanNode?
TRUE.equals(connectorExpression.orElse(TRUE)) && newDomain.contains(node.getEnforcedConstraint())) {
Expression resultingPredicate = createResultingPredicate(plannerContext, session, symbolAllocator, typeAnalyzer, splitExpression.getDynamicFilter(), TRUE_LITERAL, splitExpression.getNonDeterministicPredicate(), decomposedPredicate.getRemainingExpression());
if (!TRUE_LITERAL.equals(resultingPredicate)) {
return Optional.of(new FilterNode(filterNode.getId(), node, resultingPredicate));
}
return Optional.of(node);
}
if (newDomain.isNone()) {
// to turn the subtree into a Values node
return Optional.of(new ValuesNode(node.getId(), node.getOutputSymbols(), ImmutableList.of()));
}
Optional<ConstraintApplicationResult<TableHandle>> result = plannerContext.getMetadata().applyFilter(session, node.getTable(), constraint);
if (result.isEmpty()) {
return Optional.empty();
}
TableHandle newTable = result.get().getHandle();
TableProperties newTableProperties = plannerContext.getMetadata().getTableProperties(session, newTable);
Optional<TablePartitioning> newTablePartitioning = newTableProperties.getTablePartitioning();
if (newTableProperties.getPredicate().isNone()) {
return Optional.of(new ValuesNode(node.getId(), node.getOutputSymbols(), ImmutableList.of()));
}
TupleDomain<ColumnHandle> remainingFilter = result.get().getRemainingFilter();
Optional<ConnectorExpression> remainingConnectorExpression = result.get().getRemainingExpression();
boolean precalculateStatistics = result.get().isPrecalculateStatistics();
verifyTablePartitioning(session, plannerContext.getMetadata(), node, newTablePartitioning);
TableScanNode tableScan = new TableScanNode(node.getId(), newTable, node.getOutputSymbols(), node.getAssignments(), computeEnforced(newDomain, remainingFilter), // TODO (https://github.com/trinodb/trino/issues/8144) distinguish between predicate pushed down and remaining
deriveTableStatisticsForPushdown(statsProvider, session, precalculateStatistics, filterNode), node.isUpdateTarget(), node.getUseConnectorNodePartitioning());
Expression remainingDecomposedPredicate;
if (remainingConnectorExpression.isEmpty() || remainingConnectorExpression.equals(connectorExpression)) {
remainingDecomposedPredicate = decomposedPredicate.getRemainingExpression();
} else {
Map<String, Symbol> variableMappings = assignments.values().stream().collect(toImmutableMap(Symbol::getName, Function.identity()));
Expression translatedExpression = ConnectorExpressionTranslator.translate(session, remainingConnectorExpression.get(), plannerContext, variableMappings, new LiteralEncoder(plannerContext));
if (connectorExpression.isEmpty()) {
remainingDecomposedPredicate = ExpressionUtils.combineConjuncts(plannerContext.getMetadata(), translatedExpression, decomposedPredicate.getRemainingExpression());
} else {
remainingDecomposedPredicate = translatedExpression;
}
}
Expression resultingPredicate = createResultingPredicate(plannerContext, session, symbolAllocator, typeAnalyzer, splitExpression.getDynamicFilter(), domainTranslator.toPredicate(session, remainingFilter.transformKeys(assignments::get)), splitExpression.getNonDeterministicPredicate(), remainingDecomposedPredicate);
if (!TRUE_LITERAL.equals(resultingPredicate)) {
return Optional.of(new FilterNode(filterNode.getId(), tableScan, resultingPredicate));
}
return Optional.of(tableScan);
}
use of io.trino.sql.planner.DomainTranslator in project trino by trinodb.
the class PushdownFilterIntoWindow method apply.
@Override
public Result apply(FilterNode node, Captures captures, Context context) {
Session session = context.getSession();
TypeProvider types = context.getSymbolAllocator().getTypes();
WindowNode windowNode = captures.get(childCapture);
DomainTranslator.ExtractionResult extractionResult = DomainTranslator.getExtractionResult(plannerContext, session, node.getPredicate(), types);
TupleDomain<Symbol> tupleDomain = extractionResult.getTupleDomain();
Optional<RankingType> rankingType = toTopNRankingType(windowNode);
Symbol rankingSymbol = getOnlyElement(windowNode.getWindowFunctions().keySet());
OptionalInt upperBound = extractUpperBound(tupleDomain, rankingSymbol);
if (upperBound.isEmpty()) {
return Result.empty();
}
if (upperBound.getAsInt() <= 0) {
return Result.ofPlanNode(new ValuesNode(node.getId(), node.getOutputSymbols(), ImmutableList.of()));
}
TopNRankingNode newSource = new TopNRankingNode(windowNode.getId(), windowNode.getSource(), windowNode.getSpecification(), rankingType.get(), rankingSymbol, upperBound.getAsInt(), false, Optional.empty());
if (!allRowNumberValuesInDomain(tupleDomain, rankingSymbol, upperBound.getAsInt())) {
return Result.ofPlanNode(new FilterNode(node.getId(), newSource, node.getPredicate()));
}
// Remove the row number domain because it is absorbed into the node
TupleDomain<Symbol> newTupleDomain = tupleDomain.filter((symbol, domain) -> !symbol.equals(rankingSymbol));
Expression newPredicate = ExpressionUtils.combineConjuncts(plannerContext.getMetadata(), extractionResult.getRemainingExpression(), new DomainTranslator(plannerContext).toPredicate(session, newTupleDomain));
if (newPredicate.equals(BooleanLiteral.TRUE_LITERAL)) {
return Result.ofPlanNode(newSource);
}
return Result.ofPlanNode(new FilterNode(node.getId(), newSource, newPredicate));
}
use of io.trino.sql.planner.DomainTranslator in project trino by trinodb.
the class RemoveRedundantPredicateAboveTableScan method apply.
@Override
public Result apply(FilterNode filterNode, Captures captures, Context context) {
Session session = context.getSession();
TableScanNode node = captures.get(TABLE_SCAN);
Expression predicate = filterNode.getPredicate();
Expression deterministicPredicate = filterDeterministicConjuncts(plannerContext.getMetadata(), predicate);
Expression nonDeterministicPredicate = filterNonDeterministicConjuncts(plannerContext.getMetadata(), predicate);
ExtractionResult decomposedPredicate = getFullyExtractedPredicates(session, deterministicPredicate, context.getSymbolAllocator().getTypes());
if (decomposedPredicate.getTupleDomain().isAll()) {
// no conjunct could be fully converted to tuple domain
return Result.empty();
}
TupleDomain<ColumnHandle> predicateDomain = decomposedPredicate.getTupleDomain().transformKeys(node.getAssignments()::get);
if (predicateDomain.isNone()) {
// to turn the subtree into a Values node
return Result.ofPlanNode(new ValuesNode(node.getId(), node.getOutputSymbols(), ImmutableList.of()));
}
if (node.getEnforcedConstraint().isNone()) {
// table scans with none domain should be converted to ValuesNode
return Result.ofPlanNode(new ValuesNode(node.getId(), node.getOutputSymbols(), ImmutableList.of()));
}
// is not NONE
Map<ColumnHandle, Domain> enforcedColumnDomains = node.getEnforcedConstraint().getDomains().orElseThrow();
TupleDomain<ColumnHandle> unenforcedDomain = predicateDomain.transformDomains((columnHandle, predicateColumnDomain) -> {
Type type = predicateColumnDomain.getType();
Domain enforcedColumnDomain = Optional.ofNullable(enforcedColumnDomains.get(columnHandle)).orElseGet(() -> Domain.all(type));
if (predicateColumnDomain.contains(enforcedColumnDomain)) {
// full enforced
return Domain.all(type);
}
return predicateColumnDomain.intersect(enforcedColumnDomain);
});
if (unenforcedDomain.equals(predicateDomain)) {
// no change in filter predicate
return Result.empty();
}
Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
Expression resultingPredicate = createResultingPredicate(plannerContext, session, context.getSymbolAllocator(), typeAnalyzer, // Dynamic filters are included in decomposedPredicate.getRemainingExpression()
TRUE_LITERAL, new DomainTranslator(plannerContext).toPredicate(session, unenforcedDomain.transformKeys(assignments::get)), nonDeterministicPredicate, decomposedPredicate.getRemainingExpression());
if (!TRUE_LITERAL.equals(resultingPredicate)) {
return Result.ofPlanNode(new FilterNode(context.getIdAllocator().getNextId(), node, resultingPredicate));
}
return Result.ofPlanNode(node);
}
use of io.trino.sql.planner.DomainTranslator in project trino by trinodb.
the class ApplyTableScanRedirection method apply.
@Override
public Result apply(TableScanNode scanNode, Captures captures, Context context) {
Optional<TableScanRedirectApplicationResult> tableScanRedirectApplicationResult = plannerContext.getMetadata().applyTableScanRedirect(context.getSession(), scanNode.getTable());
if (tableScanRedirectApplicationResult.isEmpty()) {
return Result.empty();
}
CatalogSchemaTableName destinationTable = tableScanRedirectApplicationResult.get().getDestinationTable();
QualifiedObjectName destinationObjectName = convertFromSchemaTableName(destinationTable.getCatalogName()).apply(destinationTable.getSchemaTableName());
Optional<QualifiedObjectName> redirectedObjectName = plannerContext.getMetadata().getRedirectionAwareTableHandle(context.getSession(), destinationObjectName).getRedirectedTableName();
redirectedObjectName.ifPresent(name -> {
throw new TrinoException(NOT_SUPPORTED, format("Further redirection of destination table '%s' to '%s' is not supported", destinationObjectName, name));
});
TableMetadata tableMetadata = plannerContext.getMetadata().getTableMetadata(context.getSession(), scanNode.getTable());
CatalogSchemaTableName sourceTable = new CatalogSchemaTableName(tableMetadata.getCatalogName().getCatalogName(), tableMetadata.getTable());
if (destinationTable.equals(sourceTable)) {
return Result.empty();
}
Optional<TableHandle> destinationTableHandle = plannerContext.getMetadata().getTableHandle(context.getSession(), convertFromSchemaTableName(destinationTable.getCatalogName()).apply(destinationTable.getSchemaTableName()));
if (destinationTableHandle.isEmpty()) {
throw new TrinoException(TABLE_NOT_FOUND, format("Destination table %s from table scan redirection not found", destinationTable));
}
Map<ColumnHandle, String> columnMapping = tableScanRedirectApplicationResult.get().getDestinationColumns();
Map<String, ColumnHandle> destinationColumnHandles = plannerContext.getMetadata().getColumnHandles(context.getSession(), destinationTableHandle.get());
ImmutableMap.Builder<Symbol, Cast> casts = ImmutableMap.builder();
ImmutableMap.Builder<Symbol, ColumnHandle> newAssignmentsBuilder = ImmutableMap.builder();
for (Map.Entry<Symbol, ColumnHandle> assignment : scanNode.getAssignments().entrySet()) {
String destinationColumn = columnMapping.get(assignment.getValue());
if (destinationColumn == null) {
throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find mapping for source column %s in table scan redirection", assignment.getValue()));
}
ColumnHandle destinationColumnHandle = destinationColumnHandles.get(destinationColumn);
if (destinationColumnHandle == null) {
throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find handle for column %s in destination table %s", destinationColumn, destinationTable));
}
// insert ts if redirected types don't match source types
Type sourceType = context.getSymbolAllocator().getTypes().get(assignment.getKey());
Type redirectedType = plannerContext.getMetadata().getColumnMetadata(context.getSession(), destinationTableHandle.get(), destinationColumnHandle).getType();
if (!sourceType.equals(redirectedType)) {
Symbol redirectedSymbol = context.getSymbolAllocator().newSymbol(destinationColumn, redirectedType);
Cast cast = getCast(context.getSession(), destinationTable, destinationColumn, redirectedType, redirectedSymbol, sourceTable, assignment.getValue(), sourceType);
casts.put(assignment.getKey(), cast);
newAssignmentsBuilder.put(redirectedSymbol, destinationColumnHandle);
} else {
newAssignmentsBuilder.put(assignment.getKey(), destinationColumnHandle);
}
}
TupleDomain<String> requiredFilter = tableScanRedirectApplicationResult.get().getFilter();
if (requiredFilter.isAll()) {
ImmutableMap<Symbol, ColumnHandle> newAssignments = newAssignmentsBuilder.buildOrThrow();
return Result.ofPlanNode(applyProjection(context.getIdAllocator(), ImmutableSet.copyOf(scanNode.getOutputSymbols()), casts.buildOrThrow(), new TableScanNode(scanNode.getId(), destinationTableHandle.get(), ImmutableList.copyOf(newAssignments.keySet()), newAssignments, TupleDomain.all(), // Use table statistics from destination table
Optional.empty(), scanNode.isUpdateTarget(), Optional.empty())));
}
Map<ColumnHandle, Symbol> inverseAssignments = ImmutableBiMap.copyOf(scanNode.getAssignments()).inverse();
Map<String, ColumnHandle> inverseColumnsMapping = ImmutableBiMap.copyOf(columnMapping).inverse();
TupleDomain<Symbol> transformedConstraint = requiredFilter.transformKeys(destinationColumn -> {
ColumnHandle sourceColumnHandle = inverseColumnsMapping.get(destinationColumn);
if (sourceColumnHandle == null) {
throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find mapping for destination column %s in table scan redirection", destinationColumn));
}
Symbol symbol = inverseAssignments.get(sourceColumnHandle);
if (symbol != null) {
// domain symbol should already be mapped in redirected table scan
return symbol;
}
// Column pruning after predicate is pushed into table scan can remove assignments for filter columns from the scan node
Type domainType = requiredFilter.getDomains().get().get(destinationColumn).getType();
symbol = context.getSymbolAllocator().newSymbol(destinationColumn, domainType);
ColumnHandle destinationColumnHandle = destinationColumnHandles.get(destinationColumn);
if (destinationColumnHandle == null) {
throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find handle for column %s in destination table %s", destinationColumn, destinationTable));
}
// insert casts if redirected types don't match domain types
Type redirectedType = plannerContext.getMetadata().getColumnMetadata(context.getSession(), destinationTableHandle.get(), destinationColumnHandle).getType();
if (!domainType.equals(redirectedType)) {
Symbol redirectedSymbol = context.getSymbolAllocator().newSymbol(destinationColumn, redirectedType);
Cast cast = getCast(context.getSession(), destinationTable, destinationColumn, redirectedType, redirectedSymbol, sourceTable, sourceColumnHandle, domainType);
casts.put(symbol, cast);
newAssignmentsBuilder.put(redirectedSymbol, destinationColumnHandle);
} else {
newAssignmentsBuilder.put(symbol, destinationColumnHandle);
}
return symbol;
});
Map<Symbol, ColumnHandle> newAssignments = newAssignmentsBuilder.buildOrThrow();
TableScanNode newScanNode = new TableScanNode(scanNode.getId(), destinationTableHandle.get(), ImmutableList.copyOf(newAssignments.keySet()), newAssignments, TupleDomain.all(), // Use table statistics from destination table
Optional.empty(), scanNode.isUpdateTarget(), Optional.empty());
DomainTranslator domainTranslator = new DomainTranslator(plannerContext);
FilterNode filterNode = new FilterNode(context.getIdAllocator().getNextId(), applyProjection(context.getIdAllocator(), newAssignments.keySet(), casts.buildOrThrow(), newScanNode), domainTranslator.toPredicate(context.getSession(), transformedConstraint));
return Result.ofPlanNode(applyProjection(context.getIdAllocator(), ImmutableSet.copyOf(scanNode.getOutputSymbols()), ImmutableMap.of(), filterNode));
}
use of io.trino.sql.planner.DomainTranslator in project trino by trinodb.
the class PushPredicateIntoTableScan method apply.
@Override
public Result apply(FilterNode filterNode, Captures captures, Context context) {
TableScanNode tableScan = captures.get(TABLE_SCAN);
Optional<PlanNode> rewritten = pushFilterIntoTableScan(filterNode, tableScan, false, context.getSession(), context.getSymbolAllocator(), plannerContext, typeAnalyzer, context.getStatsProvider(), new DomainTranslator(plannerContext));
if (rewritten.isEmpty() || arePlansSame(filterNode, tableScan, rewritten.get())) {
return Result.empty();
}
return Result.ofPlanNode(rewritten.get());
}
Aggregations