Search in sources :

Example 1 with ConstraintApplicationResult

use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.

the class TestJmxMetadata method testApplyFilterWithSameConstraint.

@Test
public void testApplyFilterWithSameConstraint() {
    JmxTableHandle handle = metadata.getTableHandle(SESSION, new SchemaTableName(JMX_SCHEMA_NAME, "java.lang:*"));
    JmxColumnHandle columnHandle = new JmxColumnHandle("node", createUnboundedVarcharType());
    TupleDomain<ColumnHandle> nodeTupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(columnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice(localNode.getNodeIdentifier()))));
    JmxTableHandle newTableHandle = new JmxTableHandle(handle.getTableName(), handle.getObjectNames(), handle.getColumnHandles(), handle.isLiveData(), nodeTupleDomain);
    Optional<ConstraintApplicationResult<ConnectorTableHandle>> result = metadata.applyFilter(SESSION, newTableHandle, new Constraint(nodeTupleDomain));
    assertFalse(result.isPresent());
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Constraint(io.prestosql.spi.connector.Constraint) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) Test(org.testng.annotations.Test)

Example 2 with ConstraintApplicationResult

use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.

the class PushPredicateIntoTableScan method pushPredicateIntoTableScan.

/**
 * For RowExpression {@param predicate}
 */
public static Optional<PlanNode> pushPredicateIntoTableScan(TableScanNode node, RowExpression predicate, boolean pruneWithPredicateExpression, Session session, PlanNodeIdAllocator idAllocator, PlanSymbolAllocator planSymbolAllocator, Metadata metadata, RowExpressionDomainTranslator domainTranslator, boolean pushPartitionsOnly) {
    // don't include non-deterministic predicates
    LogicalRowExpressions logicalRowExpressions = new LogicalRowExpressions(new RowExpressionDeterminismEvaluator(metadata), new FunctionResolution(metadata.getFunctionAndTypeManager()), metadata.getFunctionAndTypeManager());
    RowExpression deterministicPredicate = logicalRowExpressions.filterDeterministicConjuncts(predicate);
    RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression> decomposedPredicate = domainTranslator.fromPredicate(session.toConnectorSession(), deterministicPredicate);
    TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName()))).intersect(node.getEnforcedConstraint());
    Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse();
    Set<ColumnHandle> allColumnHandles = new HashSet<>();
    assignments.keySet().stream().forEach(allColumnHandles::add);
    Constraint constraint;
    List<Constraint> disjunctConstraints = ImmutableList.of();
    if (!pushPartitionsOnly) {
        List<RowExpression> orSet = LogicalRowExpressions.extractDisjuncts(decomposedPredicate.getRemainingExpression());
        List<RowExpressionDomainTranslator.ExtractionResult<VariableReferenceExpression>> disjunctPredicates = orSet.stream().map(e -> domainTranslator.fromPredicate(session.toConnectorSession(), e)).collect(Collectors.toList());
        /* Check if any Branch yeild all records; then no need to process OR branches */
        if (!disjunctPredicates.stream().anyMatch(e -> e.getTupleDomain().isAll())) {
            List<TupleDomain<ColumnHandle>> orDomains = disjunctPredicates.stream().map(er -> er.getTupleDomain().transform(variableName -> node.getAssignments().get(new Symbol(variableName.getName())))).collect(Collectors.toList());
            disjunctConstraints = orDomains.stream().filter(d -> !d.isAll() && !d.isNone()).map(d -> new Constraint(d)).collect(Collectors.toList());
        }
    }
    if (pruneWithPredicateExpression) {
        LayoutConstraintEvaluatorForRowExpression evaluator = new LayoutConstraintEvaluatorForRowExpression(metadata, session, node.getAssignments(), logicalRowExpressions.combineConjuncts(deterministicPredicate, // which would be expensive to evaluate in the call to isCandidate below.
        domainTranslator.toPredicate(newDomain.simplify().transform(column -> {
            if (assignments.size() == 0 || assignments.getOrDefault(column, null) == null) {
                return null;
            } else {
                return new VariableReferenceExpression(assignments.getOrDefault(column, null).getName(), planSymbolAllocator.getSymbols().get(assignments.getOrDefault(column, null)));
            }
        }))));
        constraint = new Constraint(newDomain, evaluator::isCandidate);
    } else {
        // Currently, invoking the expression interpreter is very expensive.
        // TODO invoke the interpreter unconditionally when the interpreter becomes cheap enough.
        constraint = new Constraint(newDomain);
    }
    TableHandle newTable;
    TupleDomain<ColumnHandle> remainingFilter;
    if (!metadata.usesLegacyTableLayouts(session, node.getTable())) {
        if (newDomain.isNone()) {
            // to turn the subtree into a Values node
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, node.getTable(), constraint, disjunctConstraints, allColumnHandles, pushPartitionsOnly);
        if (!result.isPresent()) {
            return Optional.empty();
        }
        newTable = result.get().getHandle();
        if (metadata.getTableProperties(session, newTable).getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        remainingFilter = result.get().getRemainingFilter();
    } else {
        Optional<TableLayoutResult> layout = metadata.getLayout(session, node.getTable(), constraint, Optional.of(node.getOutputSymbols().stream().map(node.getAssignments()::get).collect(toImmutableSet())));
        if (!layout.isPresent() || layout.get().getTableProperties().getPredicate().isNone()) {
            return Optional.of(new ValuesNode(idAllocator.getNextId(), node.getOutputSymbols(), ImmutableList.of()));
        }
        newTable = layout.get().getNewTableHandle();
        remainingFilter = layout.get().getUnenforcedConstraint();
    }
    TableScanNode tableScan = new TableScanNode(node.getId(), newTable, node.getOutputSymbols(), node.getAssignments(), computeEnforced(newDomain, remainingFilter), Optional.of(deterministicPredicate), node.getStrategy(), node.getReuseTableScanMappingId(), 0, node.isForDelete());
    // The order of the arguments to combineConjuncts matters:
    // * Unenforced constraints go first because they can only be simple column references,
    // which are not prone to logic errors such as out-of-bound access, div-by-zero, etc.
    // * Conjuncts in non-deterministic expressions and non-TupleDomain-expressible expressions should
    // retain their original (maybe intermixed) order from the input predicate. However, this is not implemented yet.
    // * Short of implementing the previous bullet point, the current order of non-deterministic expressions
    // and non-TupleDomain-expressible expressions should be retained. Changing the order can lead
    // to failures of previously successful queries.
    RowExpression resultingPredicate;
    if (remainingFilter.isAll() && newTable.getConnectorHandle().hasDisjunctFiltersPushdown()) {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate));
    } else {
        resultingPredicate = logicalRowExpressions.combineConjuncts(domainTranslator.toPredicate(remainingFilter.transform(assignments::get), planSymbolAllocator.getSymbols()), logicalRowExpressions.filterNonDeterministicConjuncts(predicate), decomposedPredicate.getRemainingExpression());
    }
    if (!TRUE_CONSTANT.equals(resultingPredicate)) {
        return Optional.of(new FilterNode(idAllocator.getNextId(), tableScan, resultingPredicate));
    }
    return Optional.of(tableScan);
}
Also used : ConstantExpression(io.prestosql.spi.relation.ConstantExpression) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) TryFunction(io.prestosql.operator.scalar.TryFunction) NullableValue(io.prestosql.spi.predicate.NullableValue) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Capture.newCapture(io.prestosql.matching.Capture.newCapture) FilterNode(io.prestosql.spi.plan.FilterNode) Map(java.util.Map) Constraint(io.prestosql.spi.connector.Constraint) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) SymbolsExtractor(io.prestosql.sql.planner.SymbolsExtractor) ImmutableMap(com.google.common.collect.ImmutableMap) TableScanNode(io.prestosql.spi.plan.TableScanNode) Set(java.util.Set) PlanNode(io.prestosql.spi.plan.PlanNode) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) Collectors(java.util.stream.Collectors) Metadata(io.prestosql.metadata.Metadata) PlanSymbolAllocator(io.prestosql.sql.planner.PlanSymbolAllocator) Objects(java.util.Objects) Captures(io.prestosql.matching.Captures) RowExpressionInterpreter(io.prestosql.sql.planner.RowExpressionInterpreter) List(java.util.List) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) TableLayoutResult(io.prestosql.metadata.TableLayoutResult) Capture(io.prestosql.matching.Capture) Optional(java.util.Optional) TRUE_CONSTANT(io.prestosql.expressions.LogicalRowExpressions.TRUE_CONSTANT) Patterns.source(io.prestosql.sql.planner.plan.Patterns.source) Logger(io.airlift.log.Logger) Pattern(io.prestosql.matching.Pattern) TableHandle(io.prestosql.spi.metadata.TableHandle) Function(java.util.function.Function) TableLayoutResult.computeEnforced(io.prestosql.metadata.TableLayoutResult.computeEnforced) OPTIMIZED(io.prestosql.sql.planner.RowExpressionInterpreter.Level.OPTIMIZED) ImmutableBiMap(com.google.common.collect.ImmutableBiMap) HashSet(java.util.HashSet) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) Session(io.prestosql.Session) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) Symbol(io.prestosql.spi.plan.Symbol) Rule(io.prestosql.sql.planner.iterative.Rule) Patterns.filter(io.prestosql.sql.planner.plan.Patterns.filter) TupleDomain(io.prestosql.spi.predicate.TupleDomain) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) ValuesNode(io.prestosql.spi.plan.ValuesNode) VariableResolver(io.prestosql.sql.planner.VariableResolver) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Sets.intersection(com.google.common.collect.Sets.intersection) PlanNodeIdAllocator(io.prestosql.spi.plan.PlanNodeIdAllocator) RowExpression(io.prestosql.spi.relation.RowExpression) Patterns.tableScan(io.prestosql.sql.planner.plan.Patterns.tableScan) ValuesNode(io.prestosql.spi.plan.ValuesNode) Constraint(io.prestosql.spi.connector.Constraint) Symbol(io.prestosql.spi.plan.Symbol) FilterNode(io.prestosql.spi.plan.FilterNode) TableLayoutResult(io.prestosql.metadata.TableLayoutResult) FunctionResolution(io.prestosql.sql.relational.FunctionResolution) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) HashSet(java.util.HashSet) RowExpressionDeterminismEvaluator(io.prestosql.sql.relational.RowExpressionDeterminismEvaluator) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) LogicalRowExpressions(io.prestosql.expressions.LogicalRowExpressions) RowExpressionDomainTranslator(io.prestosql.sql.relational.RowExpressionDomainTranslator) RowExpression(io.prestosql.spi.relation.RowExpression) TupleDomain(io.prestosql.spi.predicate.TupleDomain) TableScanNode(io.prestosql.spi.plan.TableScanNode) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) TableHandle(io.prestosql.spi.metadata.TableHandle)

Example 3 with ConstraintApplicationResult

use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.

the class PushPredicateIntoUpdateDelete method pushPredicateToUpdateDelete.

private static TableHandle pushPredicateToUpdateDelete(TableHandle handle, Map<Symbol, ColumnHandle> columns, Expression predicate, Session session, TypeProvider types, Metadata metadata) {
    Expression deterministicPredicate = filterDeterministicConjuncts(predicate);
    ExpressionDomainTranslator.ExtractionResult decomposedPredicate = ExpressionDomainTranslator.fromPredicate(metadata, session, deterministicPredicate, types);
    TupleDomain<ColumnHandle> newDomain = decomposedPredicate.getTupleDomain().transform(columns::get);
    Constraint constraint = new Constraint(newDomain);
    Set<ColumnHandle> allColumnHandles = new HashSet<>();
    columns.values().stream().forEach(allColumnHandles::add);
    Optional<ConstraintApplicationResult<TableHandle>> result = metadata.applyFilter(session, handle, constraint, ImmutableList.of(), allColumnHandles, true);
    if (result.isPresent()) {
        return result.get().getHandle();
    }
    return null;
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Expression(io.prestosql.sql.tree.Expression) Constraint(io.prestosql.spi.connector.Constraint) ExpressionDomainTranslator(io.prestosql.sql.planner.ExpressionDomainTranslator) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) HashSet(java.util.HashSet)

Example 4 with ConstraintApplicationResult

use of io.prestosql.spi.connector.ConstraintApplicationResult in project boostkit-bigdata by kunpengcompute.

the class HiveMetadata method applyFilter.

@Override
public Optional<ConstraintApplicationResult<ConnectorTableHandle>> applyFilter(ConnectorSession session, ConnectorTableHandle tableHandle, Constraint constraint, List<Constraint> disjuctConstaints, Set<ColumnHandle> allColumnHandles, boolean pushPartitionsOnly) {
    HiveIdentity identity = new HiveIdentity(session);
    HiveTableHandle handle = (HiveTableHandle) tableHandle;
    checkArgument(!handle.getAnalyzePartitionValues().isPresent() || constraint.getSummary().isAll(), "Analyze should not have a constraint");
    SchemaTableName tableName = handle.getSchemaTableName();
    Table table = metastore.getTable(new HiveIdentity(session), tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    HivePartitionResult partitionResult = partitionManager.getPartitions(metastore, identity, handle, constraint, table);
    HiveTableHandle newHandle = partitionManager.applyPartitionResult(handle, partitionResult);
    // the goal here is to pushdown all the constraints/predicates to HivePageSourceProvider
    // in case some pre-filtering can be done using the heuristic-index
    // however, during scheduling we can't be sure a column will have a heuristic-index.
    // therefore, filtering should still be done using the filter operator,
    // hence the unenforced constraints below includes all constraints (minus partitions)
    ImmutableMap.Builder<HiveColumnHandle, Domain> pushedDown = ImmutableMap.builder();
    pushedDown.putAll(partitionResult.getUnenforcedConstraint().getDomains().get().entrySet().stream().collect(toMap(e -> (HiveColumnHandle) e.getKey(), e -> e.getValue())));
    TupleDomain<HiveColumnHandle> newEffectivePredicate = newHandle.getCompactEffectivePredicate().intersect(handle.getCompactEffectivePredicate()).intersect(withColumnDomains(pushedDown.build()));
    ImmutableList.Builder<TupleDomain<HiveColumnHandle>> builder = ImmutableList.builder();
    disjuctConstaints.stream().forEach(c -> {
        TupleDomain<HiveColumnHandle> newSubDomain = withColumnDomains(c.getSummary().getDomains().get().entrySet().stream().collect(toMap(e -> (HiveColumnHandle) e.getKey(), e -> e.getValue()))).subtract(newEffectivePredicate);
        if (!newSubDomain.isNone()) {
            builder.add(newSubDomain);
        }
    });
    // Get list of all columns involved in predicate
    Set<String> predicateColumnNames = new HashSet<>();
    newEffectivePredicate.getDomains().get().keySet().stream().map(HiveColumnHandle::getColumnName).forEach(predicateColumnNames::add);
    List<TupleDomain<HiveColumnHandle>> newEffectivePredicates = null;
    boolean isSuitableToPush = false;
    if (HiveSessionProperties.isOrcPredicatePushdownEnabled(session)) {
        isSuitableToPush = checkIfSuitableToPush(allColumnHandles, tableHandle, session);
    }
    if (isSuitableToPush && HiveSessionProperties.isOrcDisjunctPredicatePushdownEnabled(session)) {
        newEffectivePredicates = builder.build();
        newEffectivePredicates.stream().forEach(nfp -> nfp.getDomains().get().keySet().stream().map(HiveColumnHandle::getColumnName).forEach(predicateColumnNames::add));
    }
    if (isSuitableToPush && partitionResult.getEnforcedConstraint().equals(newEffectivePredicate) && (newEffectivePredicates == null || newEffectivePredicates.size() == 0)) {
        isSuitableToPush = false;
    }
    // Get column handle
    Map<String, ColumnHandle> columnHandles = getColumnHandles(table);
    // map predicate columns to hive column handles
    Map<String, HiveColumnHandle> predicateColumns = predicateColumnNames.stream().map(columnHandles::get).map(HiveColumnHandle.class::cast).filter(HiveColumnHandle::isRegular).collect(toImmutableMap(HiveColumnHandle::getName, identity()));
    newHandle = new HiveTableHandle(newHandle.getSchemaName(), newHandle.getTableName(), newHandle.getTableParameters(), newHandle.getPartitionColumns(), newHandle.getPartitions(), newEffectivePredicate, newHandle.getEnforcedConstraint(), newHandle.getBucketHandle(), newHandle.getBucketFilter(), newHandle.getAnalyzePartitionValues(), predicateColumns, Optional.ofNullable(newEffectivePredicates), isSuitableToPush, newHandle.getOffloadExpression());
    if (pushPartitionsOnly && handle.getPartitions().equals(newHandle.getPartitions()) && handle.getCompactEffectivePredicate().equals(newHandle.getCompactEffectivePredicate()) && handle.getBucketFilter().equals(newHandle.getBucketFilter())) {
        return Optional.empty();
    }
    if (!pushPartitionsOnly && isSuitableToPush) {
        return Optional.of(new ConstraintApplicationResult<>(newHandle, TupleDomain.all()));
    }
    // note here that all unenforced constraints will still be applied using the filter operator
    return Optional.of(new ConstraintApplicationResult<>(newHandle, partitionResult.getUnenforcedConstraint()));
}
Also used : TableStatistics(io.prestosql.spi.statistics.TableStatistics) StorageFormat(io.prestosql.plugin.hive.metastore.StorageFormat) PartialAndFinalAggregationType(io.prestosql.spi.PartialAndFinalAggregationType) FIELD_DELIM(org.apache.hadoop.hive.serde.serdeConstants.FIELD_DELIM) HiveUtil.verifyPartitionTypeSupported(io.prestosql.plugin.hive.HiveUtil.verifyPartitionTypeSupported) FileSystem(org.apache.hadoop.fs.FileSystem) HIVE_FILESYSTEM_ERROR(io.prestosql.plugin.hive.HiveErrorCode.HIVE_FILESYSTEM_ERROR) HiveUtil.hiveColumnHandles(io.prestosql.plugin.hive.HiveUtil.hiveColumnHandles) MetastoreUtil(io.prestosql.plugin.hive.metastore.MetastoreUtil) TableAlreadyExistsException(io.prestosql.spi.connector.TableAlreadyExistsException) NullableValue(io.prestosql.spi.predicate.NullableValue) RoleGrant(io.prestosql.spi.security.RoleGrant) ConnectorVacuumTableHandle(io.prestosql.spi.connector.ConnectorVacuumTableHandle) FileStatus(org.apache.hadoop.fs.FileStatus) SCHEMA_NOT_EMPTY(io.prestosql.spi.StandardErrorCode.SCHEMA_NOT_EMPTY) HiveUtil.getPartitionKeyColumnHandles(io.prestosql.plugin.hive.HiveUtil.getPartitionKeyColumnHandles) ConnectorDeleteAsInsertTableHandle(io.prestosql.spi.connector.ConnectorDeleteAsInsertTableHandle) Future(java.util.concurrent.Future) TableNotFoundException(io.prestosql.spi.connector.TableNotFoundException) BucketingVersion(io.prestosql.plugin.hive.HiveBucketing.BucketingVersion) ConnectorUpdateTableHandle(io.prestosql.spi.connector.ConnectorUpdateTableHandle) Map(java.util.Map) HiveTableProperties.getPartitionedBy(io.prestosql.plugin.hive.HiveTableProperties.getPartitionedBy) ENGLISH(java.util.Locale.ENGLISH) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) SystemTable(io.prestosql.spi.connector.SystemTable) GrantInfo(io.prestosql.spi.security.GrantInfo) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) org.apache.hadoop.hive.serde.serdeConstants(org.apache.hadoop.hive.serde.serdeConstants) TableStatisticsMetadata(io.prestosql.spi.statistics.TableStatisticsMetadata) Set(java.util.Set) LOCATION_PROPERTY(io.prestosql.plugin.hive.HiveTableProperties.LOCATION_PROPERTY) HiveTableProperties.getTransactionalValue(io.prestosql.plugin.hive.HiveTableProperties.getTransactionalValue) MANAGED_TABLE(org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE) Collectors.joining(java.util.stream.Collectors.joining) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) Table(io.prestosql.plugin.hive.metastore.Table) Privilege(io.prestosql.spi.security.Privilege) INVALID_TABLE_PROPERTY(io.prestosql.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) HiveTableProperties.isExternalTable(io.prestosql.plugin.hive.HiveTableProperties.isExternalTable) Domain(io.prestosql.spi.predicate.Domain) GENERIC_INTERNAL_ERROR(io.prestosql.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) AccessControlMetadata(io.prestosql.plugin.hive.security.AccessControlMetadata) ColumnStatisticMetadata(io.prestosql.spi.statistics.ColumnStatisticMetadata) SortingColumn(io.prestosql.plugin.hive.metastore.SortingColumn) ConnectorVacuumTableInfo(io.prestosql.spi.connector.ConnectorVacuumTableInfo) ORC(io.prestosql.plugin.hive.HiveStorageFormat.ORC) SchemaTablePrefix(io.prestosql.spi.connector.SchemaTablePrefix) Joiner(com.google.common.base.Joiner) Iterables(com.google.common.collect.Iterables) Database(io.prestosql.plugin.hive.metastore.Database) Slice(io.airlift.slice.Slice) Partition(io.prestosql.plugin.hive.metastore.Partition) TRANSACTIONAL(io.prestosql.plugin.hive.HiveTableProperties.TRANSACTIONAL) HiveWriterFactory.getSnapshotSubFileIndex(io.prestosql.plugin.hive.HiveWriterFactory.getSnapshotSubFileIndex) Supplier(java.util.function.Supplier) ComputedStatistics(io.prestosql.spi.statistics.ComputedStatistics) ArrayList(java.util.ArrayList) HiveUtil.decodeViewData(io.prestosql.plugin.hive.HiveUtil.decodeViewData) OptionalLong(java.util.OptionalLong) TupleDomain.withColumnDomains(io.prestosql.spi.predicate.TupleDomain.withColumnDomains) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ConnectorPartitioningHandle(io.prestosql.spi.connector.ConnectorPartitioningHandle) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) RecordCursor(io.prestosql.spi.connector.RecordCursor) DiscretePredicates(io.prestosql.spi.connector.DiscretePredicates) SemiTransactionalHiveMetastore(io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore) HiveWriterFactory.removeSnapshotFileName(io.prestosql.plugin.hive.HiveWriterFactory.removeSnapshotFileName) ImmutableSortedMap(com.google.common.collect.ImmutableSortedMap) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) Properties(java.util.Properties) HiveUtil.isPrestoView(io.prestosql.plugin.hive.HiveUtil.isPrestoView) TypeManager(io.prestosql.spi.type.TypeManager) IOException(java.io.IOException) USER(io.prestosql.spi.security.PrincipalType.USER) PrincipalPrivileges(io.prestosql.plugin.hive.metastore.PrincipalPrivileges) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) File(java.io.File) ExecutionException(java.util.concurrent.ExecutionException) Streams.stream(com.google.common.collect.Streams.stream) IS_EXTERNAL_TABLE(io.prestosql.plugin.hive.HiveTableProperties.IS_EXTERNAL_TABLE) HiveColumnStatistics(io.prestosql.plugin.hive.metastore.HiveColumnStatistics) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) ConnectorTablePartitioning(io.prestosql.spi.connector.ConnectorTablePartitioning) TableType(org.apache.hadoop.hive.metastore.TableType) ConfigurationUtils(io.prestosql.plugin.hive.util.ConfigurationUtils) PrestoPrincipal(io.prestosql.spi.security.PrestoPrincipal) HiveWriterFactory.isSnapshotFile(io.prestosql.plugin.hive.HiveWriterFactory.isSnapshotFile) VarcharType(io.prestosql.spi.type.VarcharType) HiveTableProperties.getLocation(io.prestosql.plugin.hive.HiveTableProperties.getLocation) ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) Statistics(io.prestosql.plugin.hive.util.Statistics) URL(java.net.URL) HdfsContext(io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext) EXTERNAL_TABLE(org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE) ViewNotFoundException(io.prestosql.spi.connector.ViewNotFoundException) HiveTableProperties.getHiveStorageFormat(io.prestosql.plugin.hive.HiveTableProperties.getHiveStorageFormat) Duration(io.airlift.units.Duration) TableStatisticType(io.prestosql.spi.statistics.TableStatisticType) FileSinkOperator(org.apache.hadoop.hive.ql.exec.FileSinkOperator) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) INVALID_ANALYZE_PROPERTY(io.prestosql.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) Collectors.toMap(java.util.stream.Collectors.toMap) ConnectorTableProperties(io.prestosql.spi.connector.ConnectorTableProperties) Iterables.concat(com.google.common.collect.Iterables.concat) Path(org.apache.hadoop.fs.Path) Type(io.prestosql.spi.type.Type) Splitter(com.google.common.base.Splitter) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) Collectors.toSet(java.util.stream.Collectors.toSet) Constraint(io.prestosql.spi.connector.Constraint) PrestoException(io.prestosql.spi.PrestoException) ImmutableSet(com.google.common.collect.ImmutableSet) HiveWriteUtils.isS3FileSystem(io.prestosql.plugin.hive.HiveWriteUtils.isS3FileSystem) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) HiveUtil.columnExtraInfo(io.prestosql.plugin.hive.HiveUtil.columnExtraInfo) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) HiveUtil.encodeViewData(io.prestosql.plugin.hive.HiveUtil.encodeViewData) HiveWriterFactory.isSnapshotSubFile(io.prestosql.plugin.hive.HiveWriterFactory.isSnapshotSubFile) ROW_COUNT(io.prestosql.spi.statistics.TableStatisticType.ROW_COUNT) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) String.format(java.lang.String.format) List(java.util.List) PRESTO_VIEW_FLAG(io.prestosql.plugin.hive.HiveUtil.PRESTO_VIEW_FLAG) VarcharType.createUnboundedVarcharType(io.prestosql.spi.type.VarcharType.createUnboundedVarcharType) Function.identity(java.util.function.Function.identity) ConnectorTransactionHandle(io.prestosql.spi.connector.ConnectorTransactionHandle) Optional(java.util.Optional) MoreObjects.firstNonNull(com.google.common.base.MoreObjects.firstNonNull) NOT_SUPPORTED(io.prestosql.spi.StandardErrorCode.NOT_SUPPORTED) AcidUtils(org.apache.hadoop.hive.ql.io.AcidUtils) HiveStatisticsProvider(io.prestosql.plugin.hive.statistics.HiveStatisticsProvider) JsonCodec(io.airlift.json.JsonCodec) IntStream(java.util.stream.IntStream) ConnectorOutputMetadata(io.prestosql.spi.connector.ConnectorOutputMetadata) Logger(io.airlift.log.Logger) ConnectorViewDefinition(io.prestosql.spi.connector.ConnectorViewDefinition) ConnectorNewTableLayout(io.prestosql.spi.connector.ConnectorNewTableLayout) HashMap(java.util.HashMap) HiveUtil.toPartitionValues(io.prestosql.plugin.hive.HiveUtil.toPartitionValues) HivePrincipal(io.prestosql.plugin.hive.metastore.HivePrincipal) OptionalInt(java.util.OptionalInt) Function(java.util.function.Function) ColumnStatisticType(io.prestosql.spi.statistics.ColumnStatisticType) InMemoryRecordSet(io.prestosql.spi.connector.InMemoryRecordSet) HashSet(java.util.HashSet) HiveTableProperties.getExternalLocation(io.prestosql.plugin.hive.HiveTableProperties.getExternalLocation) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) OpenCSVSerde(org.apache.hadoop.hive.serde2.OpenCSVSerde) INVALID_SCHEMA_PROPERTY(io.prestosql.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) ImmutableList(com.google.common.collect.ImmutableList) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) Suppliers(com.google.common.base.Suppliers) NoSuchElementException(java.util.NoSuchElementException) Block(io.prestosql.spi.block.Block) VerifyException(com.google.common.base.VerifyException) Collections.emptyMap(java.util.Collections.emptyMap) HiveIdentity(io.prestosql.plugin.hive.authentication.HiveIdentity) MalformedURLException(java.net.MalformedURLException) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) TupleDomain(io.prestosql.spi.predicate.TupleDomain) NON_INHERITABLE_PROPERTIES(io.prestosql.plugin.hive.HiveTableProperties.NON_INHERITABLE_PROPERTIES) Maps(com.google.common.collect.Maps) PRIMITIVE(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category.PRIMITIVE) JobConf(org.apache.hadoop.mapred.JobConf) Collectors.toList(java.util.stream.Collectors.toList) Column(io.prestosql.plugin.hive.metastore.Column) GENERIC_USER_ERROR(io.prestosql.spi.StandardErrorCode.GENERIC_USER_ERROR) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Comparator(java.util.Comparator) ConnectorInsertTableHandle(io.prestosql.spi.connector.ConnectorInsertTableHandle) HiveBucketing.bucketedOnTimestamp(io.prestosql.plugin.hive.HiveBucketing.bucketedOnTimestamp) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) ImmutableList(com.google.common.collect.ImmutableList) HiveIdentity(io.prestosql.plugin.hive.authentication.HiveIdentity) TableNotFoundException(io.prestosql.spi.connector.TableNotFoundException) HashSet(java.util.HashSet) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) SystemTable(io.prestosql.spi.connector.SystemTable) Table(io.prestosql.plugin.hive.metastore.Table) HiveTableProperties.isExternalTable(io.prestosql.plugin.hive.HiveTableProperties.isExternalTable) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) ImmutableMap(com.google.common.collect.ImmutableMap) TupleDomain(io.prestosql.spi.predicate.TupleDomain) Domain(io.prestosql.spi.predicate.Domain) TupleDomain(io.prestosql.spi.predicate.TupleDomain)

Example 5 with ConstraintApplicationResult

use of io.prestosql.spi.connector.ConstraintApplicationResult in project hetu-core by openlookeng.

the class TestHBaseConnector method testApplyFilter.

/**
 * testApplyFilter
 */
@Test
public void testApplyFilter() {
    Constraint constraint = new Constraint(TestUtils.createTupleDomain(5));
    Optional<ConstraintApplicationResult<ConnectorTableHandle>> result = hcm.applyFilter(session, TestUtils.createHBaseTableHandle(), constraint);
    assertEquals(true, result.isPresent());
}
Also used : Constraint(io.prestosql.spi.connector.Constraint) ConstraintApplicationResult(io.prestosql.spi.connector.ConstraintApplicationResult) Test(org.testng.annotations.Test)

Aggregations

Constraint (io.prestosql.spi.connector.Constraint)8 ConstraintApplicationResult (io.prestosql.spi.connector.ConstraintApplicationResult)8 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)6 SchemaTableName (io.prestosql.spi.connector.SchemaTableName)5 NullableValue (io.prestosql.spi.predicate.NullableValue)4 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 ImmutableList (com.google.common.collect.ImmutableList)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 ImmutableSet.toImmutableSet (com.google.common.collect.ImmutableSet.toImmutableSet)3 Logger (io.airlift.log.Logger)3 HashSet (java.util.HashSet)3 Test (org.testng.annotations.Test)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)2 Joiner (com.google.common.base.Joiner)2 MoreObjects.firstNonNull (com.google.common.base.MoreObjects.firstNonNull)2 Splitter (com.google.common.base.Splitter)2 Suppliers (com.google.common.base.Suppliers)2 Verify.verify (com.google.common.base.Verify.verify)2 VerifyException (com.google.common.base.VerifyException)2 ImmutableList.toImmutableList (com.google.common.collect.ImmutableList.toImmutableList)2