Examples with ValueSet com.facebook.presto.common.predicate.ValueSet used on opensource projects

Example 1 with ValueSet

use of com.facebook.presto.common.predicate.ValueSet in project presto by prestodb.

the class TestPrometheusSplit method testPredicatePushDownLowerBoundDirect.

@Test
public void testPredicatePushDownLowerBoundDirect() {
    Range lowRange = Range.greaterThanOrEqual(TIMESTAMP_WITH_TIME_ZONE, packDateTimeWithZone(1570460709643L, UTC_KEY));
    ValueSet valueSet = ValueSet.ofRanges(lowRange);
    Domain testDomain = Domain.create(valueSet, false);
    TupleDomain<ColumnHandle> testTupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(new PrometheusColumnHandle("timestamp", TIMESTAMP_WITH_TIME_ZONE, 2), testDomain));
    PrometheusPredicateTimeInfo predicateTimes = PrometheusSplitManager.determinePredicateTimes(testTupleDomain).orElseThrow(() -> new AssertionError("predicate pushdown error on Prometheus Column"));
    Instant expected = ofEpochMilli(1570460709643L);
    assertEquals(predicateTimes.getPredicateLowerTimeBound().orElseThrow(() -> new AssertionError("predicate pushdown error on Prometheus Column")), expected);
}

Example 2 with ValueSet

use of com.facebook.presto.common.predicate.ValueSet in project presto by prestodb.

the class ExpressionConverter method toIcebergExpression.

private static Expression toIcebergExpression(String columnName, Type type, Domain domain) {
    if (domain.isAll()) {
        return alwaysTrue();
    }
    if (domain.getValues().isNone()) {
        return domain.isNullAllowed() ? isNull(columnName) : alwaysFalse();
    }
    if (domain.getValues().isAll()) {
        return domain.isNullAllowed() ? alwaysTrue() : not(isNull(columnName));
    }
    // Skip structural types. TODO: Evaluate Apache Iceberg's support for predicate on structural types
    if (type instanceof ArrayType || type instanceof MapType || type instanceof RowType) {
        return alwaysTrue();
    }
    ValueSet domainValues = domain.getValues();
    Expression expression = null;
    if (domain.isNullAllowed()) {
        expression = isNull(columnName);
    }
    if (domainValues instanceof SortedRangeSet) {
        List<Range> orderedRanges = ((SortedRangeSet) domainValues).getOrderedRanges();
        expression = firstNonNull(expression, alwaysFalse());
        for (Range range : orderedRanges) {
            Marker low = range.getLow();
            Marker high = range.getHigh();
            Marker.Bound lowBound = low.getBound();
            Marker.Bound highBound = high.getBound();
            // case col <> 'val' is represented as (col < 'val' or col > 'val')
            if (lowBound == EXACTLY && highBound == EXACTLY) {
                // case ==
                if (getIcebergLiteralValue(type, low).equals(getIcebergLiteralValue(type, high))) {
                    expression = or(expression, equal(columnName, getIcebergLiteralValue(type, low)));
                } else {
                    // case between
                    Expression between = and(greaterThanOrEqual(columnName, getIcebergLiteralValue(type, low)), lessThanOrEqual(columnName, getIcebergLiteralValue(type, high)));
                    expression = or(expression, between);
                }
            } else {
                if (lowBound == EXACTLY && low.getValueBlock().isPresent()) {
                    // case >=
                    expression = or(expression, greaterThanOrEqual(columnName, getIcebergLiteralValue(type, low)));
                } else if (lowBound == ABOVE && low.getValueBlock().isPresent()) {
                    // case >
                    expression = or(expression, greaterThan(columnName, getIcebergLiteralValue(type, low)));
                }
                if (highBound == EXACTLY && high.getValueBlock().isPresent()) {
                    // case <=
                    if (low.getValueBlock().isPresent()) {
                        expression = and(expression, lessThanOrEqual(columnName, getIcebergLiteralValue(type, high)));
                    } else {
                        expression = or(expression, lessThanOrEqual(columnName, getIcebergLiteralValue(type, high)));
                    }
                } else if (highBound == BELOW && high.getValueBlock().isPresent()) {
                    // case <
                    if (low.getValueBlock().isPresent()) {
                        expression = and(expression, lessThan(columnName, getIcebergLiteralValue(type, high)));
                    } else {
                        expression = or(expression, lessThan(columnName, getIcebergLiteralValue(type, high)));
                    }
                }
            }
        }
        return expression;
    }
    throw new VerifyException("Did not expect a domain value set other than SortedRangeSet but got " + domainValues.getClass().getSimpleName());
}

Example 3 with ValueSet

use of com.facebook.presto.common.predicate.ValueSet in project presto by prestodb.

the class HiveSplitManager method getPartitionSplitInfo.

private Map<String, PartitionSplitInfo> getPartitionSplitInfo(ConnectorSession session, SemiTransactionalHiveMetastore metastore, SchemaTableName tableName, List<HivePartition> partitionBatch, Map<String, HiveColumnHandle> predicateColumns, Optional<Map<Subfield, Domain>> domains) {
    MetastoreContext metastoreContext = new MetastoreContext(session.getIdentity(), session.getQueryId(), session.getClientInfo(), session.getSource(), getMetastoreHeaders(session), isUserDefinedTypeEncodingEnabled(session), metastore.getColumnConverterProvider());
    Map<String, Optional<Partition>> partitions = metastore.getPartitionsByNames(metastoreContext, tableName.getSchemaName(), tableName.getTableName(), Lists.transform(partitionBatch, HivePartition::getPartitionId));
    Map<String, PartitionStatistics> partitionStatistics = ImmutableMap.of();
    if (domains.isPresent() && isPartitionStatisticsBasedOptimizationEnabled(session)) {
        partitionStatistics = metastore.getPartitionStatistics(metastoreContext, tableName.getSchemaName(), tableName.getTableName(), partitionBatch.stream().map(HivePartition::getPartitionId).collect(toImmutableSet()));
    }
    Map<String, String> partitionNameToLocation = new HashMap<>();
    ImmutableMap.Builder<String, PartitionSplitInfo> partitionSplitInfoBuilder = ImmutableMap.builder();
    for (Map.Entry<String, Optional<Partition>> entry : partitions.entrySet()) {
        ImmutableSet.Builder<ColumnHandle> redundantColumnDomainsBuilder = ImmutableSet.builder();
        if (!entry.getValue().isPresent()) {
            throw new PrestoException(HIVE_PARTITION_DROPPED_DURING_QUERY, "Partition no longer exists: " + entry.getKey());
        }
        boolean pruned = false;
        if (partitionStatistics.containsKey(entry.getKey())) {
            Map<String, HiveColumnStatistics> columnStatistics = partitionStatistics.get(entry.getKey()).getColumnStatistics();
            for (Map.Entry<String, HiveColumnHandle> predicateColumnEntry : predicateColumns.entrySet()) {
                if (columnStatistics.containsKey(predicateColumnEntry.getKey())) {
                    Optional<ValueSet> columnsStatisticsValueSet = getColumnStatisticsValueSet(columnStatistics.get(predicateColumnEntry.getKey()), predicateColumnEntry.getValue().getHiveType());
                    Subfield subfield = new Subfield(predicateColumnEntry.getKey());
                    if (columnsStatisticsValueSet.isPresent() && domains.get().containsKey(subfield)) {
                        ValueSet columnPredicateValueSet = domains.get().get(subfield).getValues();
                        if (!columnPredicateValueSet.overlaps(columnsStatisticsValueSet.get())) {
                            pruned = true;
                            break;
                        }
                        if (columnPredicateValueSet.contains(columnsStatisticsValueSet.get())) {
                            redundantColumnDomainsBuilder.add(predicateColumnEntry.getValue());
                        }
                    }
                }
            }
        }
        if (!pruned) {
            partitionNameToLocation.put(entry.getKey(), entry.getValue().get().getStorage().getLocation());
        }
        partitionSplitInfoBuilder.put(entry.getKey(), new PartitionSplitInfo(entry.getValue().get(), pruned, redundantColumnDomainsBuilder.build()));
    }
    metastore.setPartitionLeases(metastoreContext, tableName.getSchemaName(), tableName.getTableName(), partitionNameToLocation, getLeaseDuration(session));
    return partitionSplitInfoBuilder.build();
}

Example 4 with ValueSet

use of com.facebook.presto.common.predicate.ValueSet in project presto by prestodb.

the class HiveBucketing method getHiveBucketFilter.

public static Optional<HiveBucketFilter> getHiveBucketFilter(Optional<HiveBucketProperty> hiveBucketProperty, List<Column> dataColumns, TupleDomain<ColumnHandle> effectivePredicate) {
    if (!hiveBucketProperty.isPresent()) {
        return Optional.empty();
    }
    if (!hiveBucketProperty.get().getBucketFunctionType().equals(HIVE_COMPATIBLE)) {
        // bucket filtering is only supported for tables bucketed with HIVE_COMPATIBLE hash function
        return Optional.empty();
    }
    Optional<Map<ColumnHandle, Set<NullableValue>>> bindings = TupleDomain.extractFixedValueSets(effectivePredicate);
    if (!bindings.isPresent()) {
        return Optional.empty();
    }
    Optional<Set<Integer>> buckets = getHiveBuckets(hiveBucketProperty, dataColumns, bindings.get());
    if (buckets.isPresent()) {
        return Optional.of(new HiveBucketFilter(buckets.get()));
    }
    if (!effectivePredicate.getDomains().isPresent()) {
        return Optional.empty();
    }
    Optional<Domain> domain = effectivePredicate.getDomains().get().entrySet().stream().filter(entry -> ((HiveColumnHandle) entry.getKey()).getName().equals(BUCKET_COLUMN_NAME)).findFirst().map(Entry::getValue);
    if (!domain.isPresent()) {
        return Optional.empty();
    }
    ValueSet values = domain.get().getValues();
    ImmutableSet.Builder<Integer> builder = ImmutableSet.builder();
    int bucketCount = hiveBucketProperty.get().getBucketCount();
    for (int i = 0; i < bucketCount; i++) {
        if (values.containsValue((long) i)) {
            builder.add(i);
        }
    }
    return Optional.of(new HiveBucketFilter(builder.build()));
}

Example 5 with ValueSet

use of com.facebook.presto.common.predicate.ValueSet in project presto by prestodb.

the class KuduClientSession method addConstraintPredicates.

/**
 * translates TupleDomain to KuduPredicates.
 *
 * @return false if TupleDomain or one of its domains is none
 */
private boolean addConstraintPredicates(KuduTable table, KuduScanToken.KuduScanTokenBuilder builder, TupleDomain<ColumnHandle> constraintSummary) {
    if (constraintSummary.isNone()) {
        return false;
    } else if (!constraintSummary.isAll()) {
        Schema schema = table.getSchema();
        for (TupleDomain.ColumnDomain<ColumnHandle> columnDomain : constraintSummary.getColumnDomains().get()) {
            int position = ((KuduColumnHandle) columnDomain.getColumn()).getOrdinalPosition();
            ColumnSchema columnSchema = schema.getColumnByIndex(position);
            Domain domain = columnDomain.getDomain();
            if (domain.isNone()) {
                return false;
            } else if (domain.isAll()) {
            // no restriction
            } else if (domain.isOnlyNull()) {
                builder.addPredicate(KuduPredicate.newIsNullPredicate(columnSchema));
            } else if (domain.getValues().isAll() && domain.isNullAllowed()) {
                builder.addPredicate(KuduPredicate.newIsNotNullPredicate(columnSchema));
            } else if (domain.isSingleValue()) {
                KuduPredicate predicate = createEqualsPredicate(columnSchema, domain.getSingleValue());
                builder.addPredicate(predicate);
            } else {
                ValueSet valueSet = domain.getValues();
                if (valueSet instanceof EquatableValueSet) {
                    DiscreteValues discreteValues = valueSet.getDiscreteValues();
                    KuduPredicate predicate = createInListPredicate(columnSchema, discreteValues);
                    builder.addPredicate(predicate);
                } else if (valueSet instanceof SortedRangeSet) {
                    Ranges ranges = ((SortedRangeSet) valueSet).getRanges();
                    Range span = ranges.getSpan();
                    Marker low = span.getLow();
                    if (!low.isLowerUnbounded()) {
                        KuduPredicate.ComparisonOp op = (low.getBound() == Marker.Bound.ABOVE) ? KuduPredicate.ComparisonOp.GREATER : KuduPredicate.ComparisonOp.GREATER_EQUAL;
                        KuduPredicate predicate = createComparisonPredicate(columnSchema, op, low.getValue());
                        builder.addPredicate(predicate);
                    }
                    Marker high = span.getHigh();
                    if (!high.isUpperUnbounded()) {
                        KuduPredicate.ComparisonOp op = (low.getBound() == Marker.Bound.BELOW) ? KuduPredicate.ComparisonOp.LESS : KuduPredicate.ComparisonOp.LESS_EQUAL;
                        KuduPredicate predicate = createComparisonPredicate(columnSchema, op, high.getValue());
                        builder.addPredicate(predicate);
                    }
                } else {
                    throw new IllegalStateException("Unexpected domain: " + domain);
                }
            }
        }
    }
    return true;
}