Search in sources :

Example 1 with Constraint

use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.

the class TestTpchMetadata method testPartTypeAndPartContainerPredicatePushdown.

@Test
public void testPartTypeAndPartContainerPredicatePushdown() {
    TpchTableHandle tableHandle = tpchMetadata.getTableHandle(session, new SchemaTableName("sf1", PART.getTableName()));
    TupleDomain<ColumnHandle> domain;
    ConstraintApplicationResult<ConnectorTableHandle> result;
    domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("SMALL BRUSHED COPPER"));
    result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.TYPE))).get();
    assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
    assertTupleDomainEquals(PredicateUtils.filterOutColumnFromPredicate(((TpchTableHandle) result.getHandle()).getConstraint(), tpchMetadata.toColumnHandle(PartColumn.CONTAINER)), domain, session);
    domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("UNKNOWN"));
    result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.TYPE))).get();
    assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
    assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), TupleDomain.none(), session);
    domain = fixedValueTupleDomain(tpchMetadata, PartColumn.CONTAINER, utf8Slice("SM BAG"));
    result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.CONTAINER))).get();
    assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
    assertTupleDomainEquals(PredicateUtils.filterOutColumnFromPredicate(((TpchTableHandle) result.getHandle()).getConstraint(), tpchMetadata.toColumnHandle(PartColumn.TYPE)), domain, session);
    domain = fixedValueTupleDomain(tpchMetadata, PartColumn.CONTAINER, utf8Slice("UNKNOWN"));
    result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.CONTAINER))).get();
    assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
    assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), TupleDomain.none(), session);
    domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("SMALL BRUSHED COPPER"), PartColumn.CONTAINER, utf8Slice("SM BAG"));
    result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.CONTAINER))).get();
    assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
    assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), domain, session);
    domain = fixedValueTupleDomain(tpchMetadata, PartColumn.TYPE, utf8Slice("UNKNOWN"), PartColumn.CONTAINER, utf8Slice("UNKNOWN"));
    result = tpchMetadata.applyFilter(session, tableHandle, new Constraint(domain, convertToPredicate(domain, PartColumn.TYPE, PartColumn.CONTAINER))).get();
    assertTupleDomainEquals(result.getRemainingFilter(), TupleDomain.all(), session);
    assertTupleDomainEquals(((TpchTableHandle) result.getHandle()).getConstraint(), TupleDomain.none(), session);
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) Test(org.testng.annotations.Test)

Example 2 with Constraint

use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.

the class TestJmxMetadata method testApplyFilterWithSameConstraint.

@Test
public void testApplyFilterWithSameConstraint() {
    JmxTableHandle handle = metadata.getTableHandle(SESSION, new SchemaTableName(JMX_SCHEMA_NAME, "java.lang:*"));
    JmxColumnHandle columnHandle = new JmxColumnHandle("node", createUnboundedVarcharType());
    TupleDomain<ColumnHandle> nodeTupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(columnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice(localNode.getNodeIdentifier()))));
    JmxTableHandle newTableHandle = new JmxTableHandle(handle.getTableName(), handle.getObjectNames(), handle.getColumnHandles(), handle.isLiveData(), nodeTupleDomain);
    Optional<ConstraintApplicationResult<ConnectorTableHandle>> result = metadata.applyFilter(SESSION, newTableHandle, new Constraint(nodeTupleDomain));
    assertFalse(result.isPresent());
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) Test(org.testng.annotations.Test)

Example 3 with Constraint

use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.

the class TableScanStatsRule method doCalculate.

@Override
protected Optional<PlanNodeStatsEstimate> doCalculate(TableScanNode node, StatsProvider sourceStats, Lookup lookup, Session session, TypeProvider types) {
    // TODO Construct predicate like AddExchanges's LayoutConstraintEvaluator
    TupleDomain<ColumnHandle> predicate = metadata.getTableProperties(session, node.getTable()).getPredicate();
    Constraint constraint = new Constraint(predicate);
    TableStatistics tableStatistics = metadata.getTableStatistics(session, node.getTable(), constraint, true);
    verify(tableStatistics != null, "tableStatistics is null for %s", node);
    Map<Symbol, SymbolStatsEstimate> outputSymbolStats = new HashMap<>();
    Map<ColumnHandle, Symbol> remainingSymbols = new HashMap<>();
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    boolean isPredicatesPushDown = false;
    if ((predicate.isAll() || predicate.getDomains().get().equals(node.getEnforcedConstraint().getDomains().get())) && !(node.getEnforcedConstraint().isAll() || node.getEnforcedConstraint().isNone())) {
        predicate = node.getEnforcedConstraint();
        isPredicatesPushDown = true;
        predicate.getDomains().get().entrySet().stream().forEach(e -> {
            remainingSymbols.put(e.getKey(), new Symbol(e.getKey().getColumnName()));
        });
    }
    for (Map.Entry<Symbol, ColumnHandle> entry : node.getAssignments().entrySet()) {
        Symbol symbol = entry.getKey();
        Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getValue()));
        SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
        outputSymbolStats.put(symbol, symbolStatistics);
        remainingSymbols.remove(entry.getValue());
    }
    PlanNodeStatsEstimate tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
    if (isPredicatesPushDown) {
        if (remainingSymbols.size() > 0) {
            ImmutableBiMap.Builder<ColumnHandle, Symbol> assignmentBuilder = ImmutableBiMap.builder();
            assignments = assignmentBuilder.putAll(assignments).putAll(remainingSymbols).build();
            for (Map.Entry<ColumnHandle, Symbol> entry : remainingSymbols.entrySet()) {
                Symbol symbol = entry.getValue();
                Optional<ColumnStatistics> columnStatistics = Optional.ofNullable(tableStatistics.getColumnStatistics().get(entry.getKey()));
                SymbolStatsEstimate symbolStatistics = columnStatistics.map(statistics -> toSymbolStatistics(tableStatistics, statistics, types.get(symbol))).orElse(SymbolStatsEstimate.unknown());
                outputSymbolStats.put(symbol, symbolStatistics);
            }
            /* Refresh TableEstimates for remaining columns */
            tableEstimates = PlanNodeStatsEstimate.builder().setOutputRowCount(tableStatistics.getRowCount().getValue()).addSymbolStatistics(outputSymbolStats).build();
        }
        Expression pushDownExpression = domainTranslator.toPredicate(predicate.transform(assignments::get));
        PlanNodeStatsEstimate estimate = filterStatsCalculator.filterStats(tableEstimates, pushDownExpression, session, types);
        if ((isDefaultFilterFactorEnabled(session) || sourceStats.isEnforceDefaultFilterFactor()) && estimate.isOutputRowCountUnknown()) {
            PlanNodeStatsEstimate finalTableEstimates = tableEstimates;
            estimate = tableEstimates.mapOutputRowCount(sourceRowCount -> finalTableEstimates.getOutputRowCount() * UNKNOWN_FILTER_COEFFICIENT);
        }
        return Optional.of(estimate);
    }
    return Optional.of(tableEstimates);
}
Also used : ColumnStatistics(io.prestosql.spi.statistics.ColumnStatistics) TableStatistics(io.prestosql.spi.statistics.TableStatistics) Lookup(io.prestosql.sql.planner.iterative.Lookup) TypeProvider(io.prestosql.sql.planner.TypeProvider) HashMap(java.util.HashMap) Pattern(io.prestosql.matching.Pattern) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) NaN(java.lang.Double.NaN) ExpressionDomainTranslator(io.prestosql.sql.planner.ExpressionDomainTranslator) Verify.verify(com.google.common.base.Verify.verify) FixedWidthType(io.prestosql.spi.type.FixedWidthType) Map(java.util.Map) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) Type(io.prestosql.spi.type.Type) ColumnStatistics(io.prestosql.spi.statistics.ColumnStatistics) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) UNKNOWN_FILTER_COEFFICIENT(io.prestosql.cost.FilterStatsCalculator.UNKNOWN_FILTER_COEFFICIENT) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TableScanNode(io.prestosql.spi.plan.TableScanNode) SystemSessionProperties.isDefaultFilterFactorEnabled(io.prestosql.SystemSessionProperties.isDefaultFilterFactorEnabled) Metadata(io.prestosql.metadata.Metadata) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LiteralEncoder(io.prestosql.sql.planner.LiteralEncoder) Optional(java.util.Optional) Patterns.tableScan(io.prestosql.sql.planner.plan.Patterns.tableScan) Expression(io.prestosql.sql.tree.Expression) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) HashMap(java.util.HashMap) Symbol(io.prestosql.spi.plan.Symbol) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Expression(io.prestosql.sql.tree.Expression) TableStatistics(io.prestosql.spi.statistics.TableStatistics) HashMap(java.util.HashMap) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) Map(java.util.Map)

Example 4 with Constraint

use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.

the class TableDeleteOperator method getOutput.

@Override
public Page getOutput() {
    if (state == State.FINISHED || (sourceLayout.isPresent() && state != State.FINISHING)) {
        return null;
    }
    state = State.FINISHED;
    TableHandle tableHandleForDelete = tableHandle;
    if (sourceLayout.isPresent()) {
        // Replace the values from subqueries in filter predicates
        VariablesExtractor.extractUnique(filter.get()).stream().forEach(expr -> symbolExpressionMap.putIfAbsent(new Symbol(expr.getName()), expr));
        RowExpression rowExpression = RowExpressionVariableInliner.inlineVariables(node -> symbolExpressionMap.get(new Symbol(node.getName())), filter.get());
        // Create the tuple domain based on filter and values from source;
        RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = (new RowExpressionDomainTranslator(metadata)).fromPredicate(session.toConnectorSession(), rowExpression, RowExpressionDomainTranslator.BASIC_COLUMN_EXTRACTOR);
        TupleDomain<ColumnHandle> tupleDomain = decomposedPredicate.getTupleDomain().transform(variableName -> assignments.get(new Symbol(variableName.getName())));
        Constraint constraint = new Constraint(tupleDomain);
        // Apply the constraint on the table handle
        Optional<TableHandle> tableHandleDelete = metadata.applyDelete(session, this.tableHandle, constraint);
        if (tableHandleDelete.isPresent()) {
            tableHandleForDelete = tableHandleDelete.get();
        }
    }
    OptionalLong rowsDeletedCount = metadata.executeDelete(session, tableHandleForDelete);
    // output page will only be constructed once,
    // so a new PageBuilder is constructed (instead of using PageBuilder.reset)
    PageBuilder page = new PageBuilder(1, TYPES);
    BlockBuilder rowsBuilder = page.getBlockBuilder(0);
    page.declarePosition();
    if (rowsDeletedCount.isPresent()) {
        BIGINT.writeLong(rowsBuilder, rowsDeletedCount.getAsLong());
    } else {
        rowsBuilder.appendNull();
    }
    return page.build();
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) RowExpression(io.prestosql.spi.relation.RowExpression) PageBuilder(io.prestosql.spi.PageBuilder) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) OptionalLong(java.util.OptionalLong) TableHandle(io.prestosql.spi.metadata.TableHandle) BlockBuilder(io.prestosql.spi.block.BlockBuilder)

Example 5 with Constraint

use of io.prestosql.spi.connector.Constraint in project hetu-core by openlookeng.

the class AbstractTestHive method testPartitionSchemaNonCanonical.

// TODO coercion of non-canonical values should be supported
@Test(enabled = false)
public void testPartitionSchemaNonCanonical() throws Exception {
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle table = getTableHandle(metadata, tablePartitionSchemaChangeNonCanonical);
        ColumnHandle column = metadata.getColumnHandles(session, table).get("t_boolean");
        Constraint constraint = new Constraint(TupleDomain.fromFixedValues(ImmutableMap.of(column, NullableValue.of(BOOLEAN, false))));
        table = applyFilter(metadata, table, constraint);
        HivePartition partition = getOnlyElement(((HiveTableHandle) table).getPartitions().orElseThrow(AssertionError::new));
        assertEquals(getPartitionId(partition), "t_boolean=0");
        ConnectorSplitSource splitSource = splitManager.getSplits(transaction.getTransactionHandle(), session, table, UNGROUPED_SCHEDULING);
        ConnectorSplit split = getOnlyElement(getAllSplits(splitSource));
        ImmutableList<ColumnHandle> columnHandles = ImmutableList.of(column);
        try (ConnectorPageSource ignored = pageSourceProvider.createPageSource(transaction.getTransactionHandle(), session, split, table, columnHandles)) {
            fail("expected exception");
        } catch (PrestoException e) {
            assertEquals(e.getErrorCode(), HiveErrorCode.HIVE_INVALID_PARTITION_VALUE.toErrorCode());
        }
    }
}
Also used : HiveColumnHandle.bucketColumnHandle(io.prestosql.plugin.hive.HiveColumnHandle.bucketColumnHandle) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) PrestoException(io.prestosql.spi.PrestoException) ConnectorSplitSource(io.prestosql.spi.connector.ConnectorSplitSource) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) Test(org.testng.annotations.Test)

Aggregations

Constraint (io.prestosql.spi.connector.Constraint)26 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)20 SchemaTableName (io.prestosql.spi.connector.SchemaTableName)17 ConnectorTableHandle (io.prestosql.spi.connector.ConnectorTableHandle)15 ConstraintApplicationResult (io.prestosql.spi.connector.ConstraintApplicationResult)14 ConnectorMetadata (io.prestosql.spi.connector.ConnectorMetadata)13 ConnectorSession (io.prestosql.spi.connector.ConnectorSession)13 TupleDomain (io.prestosql.spi.predicate.TupleDomain)13 Test (org.testng.annotations.Test)13 NullableValue (io.prestosql.spi.predicate.NullableValue)12 ImmutableList (com.google.common.collect.ImmutableList)11 ImmutableMap (com.google.common.collect.ImmutableMap)11 Domain (io.prestosql.spi.predicate.Domain)11 Map (java.util.Map)11 Objects.requireNonNull (java.util.Objects.requireNonNull)11 Optional (java.util.Optional)11 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)10 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)10 Slice (io.airlift.slice.Slice)10 List (java.util.List)10