use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.
the class TestJmxMetadata method testApplyFilterWithSameConstraint.
@Test
public void testApplyFilterWithSameConstraint() {
JmxTableHandle handle = metadata.getTableHandle(SESSION, new SchemaTableName(JMX_SCHEMA_NAME, "java.lang:*"));
JmxColumnHandle columnHandle = new JmxColumnHandle("node", createUnboundedVarcharType());
TupleDomain<ColumnHandle> nodeTupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(columnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice(localNode.getNodeIdentifier()))));
JmxTableHandle newTableHandle = new JmxTableHandle(handle.getTableName(), handle.getObjectNames(), handle.getColumnHandles(), handle.isLiveData(), nodeTupleDomain);
Optional<ConstraintApplicationResult<ConnectorTableHandle>> result = metadata.applyFilter(SESSION, newTableHandle, new Constraint(nodeTupleDomain));
assertFalse(result.isPresent());
}
use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.
the class PushPredicateIntoTableScan method pushPredicateIntoTableScan.
/**
* For RowExpression {@param predicate}
*/
public static Optional<PlanNode> pushPredicateIntoTableScan(TableScanNode node, RowExpression predicate, boolean pruneWithPredicateExpression, Session session, PlanNodeIdAllocator idAllocator, PlanSymbolAllocator planSymbolAllocator, Metadata metadata, RowExpressionDomainTranslator domainTranslator, boolean pushPartitionsOnly) {
// don't include non-deterministic predicates
LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(metadata), new FunctionResolution(metadata.getFunctionAndTypeManager()), metadata.getFunctionAndTypeManager());
RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(predicate);
RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = domainTranslator.fromPredicate(session.toConnectorSession(), deterministicPredicate);
TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName()))).intersect(node.getEnforcedConstraint());
Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
Set<ColumnHandle> allColumnHandles = new HashSet<>();
assignments.keySet().stream().forEach(allColumnHandles::add);
Constraint constraint;
List<Constraint> disjunctConstraints = ImmutableList.of();
if (!pushPartitionsOnly) {
List<RowExpression> orSet = LogicalRowExpressions.extractDisjuncts(decomposedPredicate.getRemainingExpression());
List<RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression>> disjunctPredicates = orSet.stream().map(e -> domainTranslator.fromPredicate(session.toConnectorSession(), e)).collect(Collectors.toList());
/* Check if any Branch yeild all records; then no need to process OR branches */
if (!disjunctPredicates.stream().anyMatch(e -> e.getTupleDomain().isAll())) {
List<TupleDomain<ColumnHandle>> orDomains = disjunctPredicates.stream().map(er -> er.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName())))).collect(Collectors.toList());
disjunctConstraints = orDomains.stream().filter(d -> !d.isAll() && !d.isNone()).map(d -> new Constraint(d)).collect(Collectors.toList());
}
}
if (pruneWithPredicateExpression) {
LayoutConstraintEvaluatorForRowExpression evaluator = new LayoutConstraintEvaluatorForRowExpression(metadata, session, node.getAssignments(), logicalRowExpressions.combineConjuncts(deterministicPredicate, // which would be expensive to evaluate in the call to isCandidate below.
domainTranslator.toPredicate(newDomain.simplify().transform(column -> {
if (assignments.size() == 0 || assignments.getOrDefault(column, null) == null) {
return null;
} else {
return new VariableReferenceExpression(assignments.getOrDefault(column, null).getName(), planSymbolAllocator.getSymbols().get(assignments.getOrDefault(column, null)));
}
}))));
constraint = new Constraint(newDomain, evaluator::isCandidate);
} else {
// Currently, invoking the expression interpreter is very expensive.
// TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough.
constraint = new Constraint(newDomain);
}
TableHandle newTable;
TupleDomain<ColumnHandle> remainingFilter;
if (!metadata.usesLegacyTableLayouts(session, node.getTable())) {
if (newDomain.isNone()) {
// to turn the subtree into a Values node
return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
}
Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, node.getTable(), constraint, disjunctConstraints, allColumnHandles, pushPartitionsOnly);
if (!result.isPresent()) {
return Optional.empty();
}
newTable = result.get().getHandle();
if (metadata.getTableProperties(session, newTable).getPredicate().isNone()) {
return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
}
remainingFilter = result.get().getRemainingFilter();
} else {
Optional<TableLayoutResult> layout = metadata.getLayout(session, node.getTable(), constraint, Optional.of(node.getOutputSymbols().stream().map(node.getAssignments()::get).collect(toImmutableSet())));
if (!layout.isPresent() || layout.get().getTableProperties().getPredicate().isNone()) {
return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
}
newTable = layout.get().getNewTableHandle();
remainingFilter = layout.get().getUnenforcedConstraint();
}
TableScanNode tableScan = new TableScanNode(node.getId(), newTable, node.getOutputSymbols(), node.getAssignments(), computeEnforced(newDomain, remainingFilter), Optional.of(deterministicPredicate), node.getStrategy(), node.getReuseTableScanMappingId(), 0, node.isForDelete());
// The order of the arguments to combineConjuncts matters:
// * Unenforced constraints go first because they can only be simple column references,
// which are not prone to logic errors such as out-of-bound access, div-by-zero, etc.
// * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should
// retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet.
// * Short of implementing the previous bullet point, the current order of non-deterministic expressions
// and non-TupleDomain-expressible expressions should be retained. Changing the order can lead
// to failures of previously successful queries.
RowExpression resultingPredicate;
if (remainingFilter.isAll() && newTable.getConnectorHandle().hasDisjunctFiltersPushdown()) {
resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate));
} else {
resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate), decomposedPredicate.getRemainingExpression());
}
if (!TRUE_CONSTANT.equals(resultingPredicate)) {
return Optional.of(new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate));
}
return Optional.of(tableScan);
}
use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.
the class PushPredicateIntoUpdateDelete method pushPredicateToUpdateDelete.
private static TableHandle pushPredicateToUpdateDelete(TableHandle handle, Map<Symbol, ColumnHandle> columns, Expression predicate, Session session, TypeProvider types, Metadata metadata) {
Expression deterministicPredicate = filterDeterministicConjuncts(predicate);
ExpressionDomainTranslator.ExtractionResult decomposedPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, deterministicPredicate, types);
TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transform(columns::get);
Constraint constraint = new Constraint(newDomain);
Set<ColumnHandle> allColumnHandles = new HashSet<>();
columns.values().stream().forEach(allColumnHandles::add);
Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, handle, constraint, ImmutableList.of(), allColumnHandles, true);
if (result.isPresent()) {
return result.get().getHandle();
}
return null;
}
use of io.prestosql.spi.connector.ConstraintApplicationResult in project boostkit-bigdata by kunpengcompute.
the class HiveMetadata method applyFilter.
@Override
public Optional<ConstraintApplicationResult<ConnectorTableHandle>> applyFilter(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint, List<Constraint> disjuctConstaints, Set<ColumnHandle> allColumnHandles, boolean pushPartitionsOnly) {
HiveIdentity identity = new HiveIdentity(session);
HiveTableHandle handle = (HiveTableHandle) tableHandle;
checkArgument(!handle.getAnalyzePartitionValues().isPresent() || constraint.getSummary().isAll(), "Analyze should not have a constraint");
SchemaTableName tableName = handle.getSchemaTableName();
Table table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
HivePartitionResult partitionResult = partitionManager.getPartitions(metastore, identity, handle, constraint, table);
HiveTableHandle newHandle = partitionManager.applyPartitionResult(handle, partitionResult);
// the goal here is to pushdown all the constraints/predicates to HivePageSourceProvider
// in case some pre-filtering can be done using the heuristic-index
// however, during scheduling we can't be sure a column will have a heuristic-index.
// therefore, filtering should still be done using the filter operator,
// hence the unenforced constraints below includes all constraints (minus partitions)
ImmutableMap.Builder<HiveColumnHandle, Domain> pushedDown = ImmutableMap.builder();
pushedDown.putAll(partitionResult.getUnenforcedConstraint().getDomains().get().entrySet().stream().collect(toMap(e -> (HiveColumnHandle) e.getKey(), e -> e.getValue())));
TupleDomain<HiveColumnHandle> newEffectivePredicate = newHandle.getCompactEffectivePredicate().intersect(handle.getCompactEffectivePredicate()).intersect(withColumnDomains(pushedDown.build()));
ImmutableList.Builder<TupleDomain<HiveColumnHandle>> builder = ImmutableList.builder();
disjuctConstaints.stream().forEach(c -> {
TupleDomain<HiveColumnHandle> newSubDomain = withColumnDomains(c.getSummary().getDomains().get().entrySet().stream().collect(toMap(e -> (HiveColumnHandle) e.getKey(), e -> e.getValue()))).subtract(newEffectivePredicate);
if (!newSubDomain.isNone()) {
builder.add(newSubDomain);
}
});
// Get list of all columns involved in predicate
Set<String> predicateColumnNames = new HashSet<>();
newEffectivePredicate.getDomains().get().keySet().stream().map(HiveColumnHandle::getColumnName).forEach(predicateColumnNames::add);
List<TupleDomain<HiveColumnHandle>> newEffectivePredicates = null;
boolean isSuitableToPush = false;
if (HiveSessionProperties.isOrcPredicatePushdownEnabled(session)) {
isSuitableToPush = checkIfSuitableToPush(allColumnHandles, tableHandle, session);
}
if (isSuitableToPush && HiveSessionProperties.isOrcDisjunctPredicatePushdownEnabled(session)) {
newEffectivePredicates = builder.build();
newEffectivePredicates.stream().forEach(nfp -> nfp.getDomains().get().keySet().stream().map(HiveColumnHandle::getColumnName).forEach(predicateColumnNames::add));
}
if (isSuitableToPush && partitionResult.getEnforcedConstraint().equals(newEffectivePredicate) && (newEffectivePredicates == null || newEffectivePredicates.size() == 0)) {
isSuitableToPush = false;
}
// Get column handle
Map<String, ColumnHandle> columnHandles = getColumnHandles(table);
// map predicate columns to hive column handles
Map<String, HiveColumnHandle> predicateColumns = predicateColumnNames.stream().map(columnHandles::get).map(HiveColumnHandle.class::cast).filter(HiveColumnHandle::isRegular).collect(toImmutableMap(HiveColumnHandle::getName, identity()));
newHandle = new HiveTableHandle(newHandle.getSchemaName(), newHandle.getTableName(), newHandle.getTableParameters(), newHandle.getPartitionColumns(), newHandle.getPartitions(), newEffectivePredicate, newHandle.getEnforcedConstraint(), newHandle.getBucketHandle(), newHandle.getBucketFilter(), newHandle.getAnalyzePartitionValues(), predicateColumns, Optional.ofNullable(newEffectivePredicates), isSuitableToPush, newHandle.getOffloadExpression());
if (pushPartitionsOnly && handle.getPartitions().equals(newHandle.getPartitions()) && handle.getCompactEffectivePredicate().equals(newHandle.getCompactEffectivePredicate()) && handle.getBucketFilter().equals(newHandle.getBucketFilter())) {
return Optional.empty();
}
if (!pushPartitionsOnly && isSuitableToPush) {
return Optional.of(new ConstraintApplicationResult<>(newHandle, TupleDomain.all()));
}
// note here that all unenforced constraints will still be applied using the filter operator
return Optional.of(new ConstraintApplicationResult<>(newHandle, partitionResult.getUnenforcedConstraint()));
}
use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.
the class TestHBaseConnector method testApplyFilter.
/**
* testApplyFilter
*/
@Test
public void testApplyFilter() {
Constraint constraint = new Constraint(TestUtils.createTupleDomain(5));
Optional<ConstraintApplicationResult<ConnectorTableHandle>> result = hcm.applyFilter(session, TestUtils.createHBaseTableHandle(), constraint);
assertEquals(true, result.isPresent());
}
Aggregations