use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.
the class TestTpchMetadata method testPartTypeAndPartContainerPredicatePushdown.
@Test
public void testPartTypeAndPartContainerPredicatePushdown() {
TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", PART.getTableName()));
TupleDomain<ColumnHandle> domain;
ConstraintApplicationResult<ConnectorTableHandle> result;
domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("SMALL BRUSHED COPPER"));
result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.TYPE))).get();
assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
assertTupleDomainEquals(PredicateUtils.filterOutColumnFromPredicate(((TpchTableHandle) result.getHandle()).getConstraint(), tpchMetadata.toColumnHandle(PartColumn.CONTAINER)), domain, session);
domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("UNKNOWN"));
result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.TYPE))).get();
assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), TupleDomain.none(), session);
domain = fixedValueTupleDomain(tpchMetadata, PartColumn.CONTAINER, utf8Slice("SM BAG"));
result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.CONTAINER))).get();
assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
assertTupleDomainEquals(PredicateUtils.filterOutColumnFromPredicate(((TpchTableHandle) result.getHandle()).getConstraint(), tpchMetadata.toColumnHandle(PartColumn.TYPE)), domain, session);
domain = fixedValueTupleDomain(tpchMetadata, PartColumn.CONTAINER, utf8Slice("UNKNOWN"));
result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.CONTAINER))).get();
assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), TupleDomain.none(), session);
domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("SMALL BRUSHED COPPER"), PartColumn.CONTAINER, utf8Slice("SM BAG"));
result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.CONTAINER))).get();
assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), domain, session);
domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("UNKNOWN"), PartColumn.CONTAINER, utf8Slice("UNKNOWN"));
result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.TYPE, PartColumn.CONTAINER))).get();
assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), TupleDomain.none(), session);
}
use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.
the class TestJmxMetadata method testApplyFilterWithSameConstraint.
@Test
public void testApplyFilterWithSameConstraint() {
JmxTableHandle handle = metadata.getTableHandle(SESSION, new SchemaTableName(JMX_SCHEMA_NAME, "java.lang:*"));
JmxColumnHandle columnHandle = new JmxColumnHandle("node", createUnboundedVarcharType());
TupleDomain<ColumnHandle> nodeTupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(columnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice(localNode.getNodeIdentifier()))));
JmxTableHandle newTableHandle = new JmxTableHandle(handle.getTableName(), handle.getObjectNames(), handle.getColumnHandles(), handle.isLiveData(), nodeTupleDomain);
Optional<ConstraintApplicationResult<ConnectorTableHandle>> result = metadata.applyFilter(SESSION, newTableHandle, new Constraint(nodeTupleDomain));
assertFalse(result.isPresent());
}
use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.
the class TableScanStatsRule method doCalculate.
@Override
protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) {
// TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator
TupleDomain<ColumnHandle> predicate = metadata.getTableProperties(session, node.getTable()).getPredicate();
Constraint constraint = new Constraint(predicate);
TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint, true);
verify(tableStatistics != null, "tableStatistics is null for %s", node);
Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>();
Map<ColumnHandle, Symbol> remainingSymbols = new HashMap<>();
Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
boolean isPredicatesPushDown = false;
if ((predicate.isAll() || predicate.getDomains().get().equals(node.getEnforcedConstraint().getDomains().get())) && !(node.getEnforcedConstraint().isAll() || node.getEnforcedConstraint().isNone())) {
predicate = node.getEnforcedConstraint();
isPredicatesPushDown = true;
predicate.getDomains().get().entrySet().stream().forEach(e -> {
remainingSymbols.put(e.getKey(), new Symbol(e.getKey().getColumnName()));
});
}
for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) {
Symbol symbol = entry.getKey();
Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue()));
SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
outputSymbolStats.put(symbol, symbolStatistics);
remainingSymbols.remove(entry.getValue());
}
PlanNodeStatsEstimate tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
if (isPredicatesPushDown) {
if (remainingSymbols.size() > 0) {
ImmutableBiMap.Builder<ColumnHandle, Symbol> assignmentBuilder = ImmutableBiMap.builder();
assignments = assignmentBuilder.putAll(assignments).putAll(remainingSymbols).build();
for (Map.Entry<ColumnHandle, Symbol> entry : remainingSymbols.entrySet()) {
Symbol symbol = entry.getValue();
Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getKey()));
SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
outputSymbolStats.put(symbol, symbolStatistics);
}
/* Refresh TableEstimates for remaining columns */
tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
}
Expression pushDownExpression = domainTranslator.toPredicate(predicate.transform(assignments::get));
PlanNodeStatsEstimate estimate = filterStatsCalculator.filterStats(tableEstimates, pushDownExpression, session, types);
if ((isDefaultFilterFactorEnabled(session) || sourceStats.isEnforceDefaultFilterFactor()) && estimate.isOutputRowCountUnknown()) {
PlanNodeStatsEstimate finalTableEstimates = tableEstimates;
estimate = tableEstimates.mapOutputRowCount(sourceRowCount -> finalTableEstimates.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT);
}
return Optional.of(estimate);
}
return Optional.of(tableEstimates);
}
use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.
the class TableDeleteOperator method getOutput.
@Override
public Page getOutput() {
if (state == State.FINISHED || (sourceLayout.isPresent() && state != State.FINISHING)) {
return null;
}
state = State.FINISHED;
TableHandle tableHandleForDelete = tableHandle;
if (sourceLayout.isPresent()) {
// Replace the values from subqueries in filter predicates
VariablesExtractor.extractUnique(filter.get()).stream().forEach(expr -> symbolExpressionMap.putIfAbsent(new Symbol(expr.getName()), expr));
RowExpression rowExpression = RowExpressionVariableInliner.inlineVariables(node -> symbolExpressionMap.get(new Symbol(node.getName())), filter.get());
// Create the tuple domain based on filter and values from source;
RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = (new RowExpressionDomainTranslator(metadata)).fromPredicate(session.toConnectorSession(), rowExpression, RowExpressionDomainTranslator.BASIC_COLUMN_EXTRACTOR);
TupleDomain<ColumnHandle> tupleDomain = decomposedPredicate.getTupleDomain().transform(variableName -> assignments.get(new Symbol(variableName.getName())));
Constraint constraint = new Constraint(tupleDomain);
// Apply the constraint on the table handle
Optional<TableHandle> tableHandleDelete = metadata.applyDelete(session, this.tableHandle, constraint);
if (tableHandleDelete.isPresent()) {
tableHandleForDelete = tableHandleDelete.get();
}
}
OptionalLong rowsDeletedCount = metadata.executeDelete(session, tableHandleForDelete);
// output page will only be constructed once,
// so a new PageBuilder is constructed (instead of using PageBuilder.reset)
PageBuilder page = new PageBuilder(1, TYPES);
BlockBuilder rowsBuilder = page.getBlockBuilder(0);
page.declarePosition();
if (rowsDeletedCount.isPresent()) {
BIGINT.writeLong(rowsBuilder, rowsDeletedCount.getAsLong());
} else {
rowsBuilder.appendNull();
}
return page.build();
}
use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.
the class AbstractTestHive method testPartitionSchemaNonCanonical.
// TODO coercion of non-canonical values should be supported
@Test(enabled = false)
public void testPartitionSchemaNonCanonical() throws Exception {
try (Transaction transaction = newTransaction()) {
ConnectorSession session = newSession();
ConnectorMetadata metadata = transaction.getMetadata();
ConnectorTableHandle table = getTableHandle(metadata, tablePartitionSchemaChangeNonCanonical);
ColumnHandle column = metadata.getColumnHandles(session, table).get("t_boolean");
Constraint constraint = new Constraint(TupleDomain.fromFixedValues(ImmutableMap.of(column, NullableValue.of(BOOLEAN, false))));
table = applyFilter(metadata, table, constraint);
HivePartition partition = getOnlyElement(((HiveTableHandle) table).getPartitions().orElseThrow(AssertionError::new));
assertEquals(getPartitionId(partition), "t_boolean=0");
ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, table, UNGROUPED_SCHEDULING);
ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
ImmutableList<ColumnHandle> columnHandles = ImmutableList.of(column);
try (ConnectorPageSource ignored = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, table, columnHandles)) {
fail("expected exception");
} catch (PrestoException e) {
assertEquals(e.getErrorCode(), HiveErrorCode.HIVE_INVALID_PARTITION_VALUE.toErrorCode());
}
}
}
Aggregations