Examples with TupleDomain com.facebook.presto.spi.predicate.TupleDomain used on opensource projects

Example 1 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class HivePartitionManager method getPartitions.

public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastore, ConnectorTableHandle tableHandle, Constraint<ColumnHandle> constraint) {
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    TupleDomain<ColumnHandle> effectivePredicate = constraint.getSummary();
    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    Table table = getTable(metastore, tableName);
    Optional<HiveBucketHandle> hiveBucketHandle = getHiveBucketHandle(connectorId, table);
    List<HiveColumnHandle> partitionColumns = getPartitionKeyColumnHandles(connectorId, table);
    List<HiveBucket> buckets = getHiveBucketNumbers(table, effectivePredicate);
    TupleDomain<HiveColumnHandle> compactEffectivePredicate = toCompactTupleDomain(effectivePredicate, domainCompactionThreshold);
    if (effectivePredicate.isNone()) {
        return new HivePartitionResult(partitionColumns, ImmutableList.of(), TupleDomain.none(), TupleDomain.none(), hiveBucketHandle);
    }
    if (partitionColumns.isEmpty()) {
        return new HivePartitionResult(partitionColumns, ImmutableList.of(new HivePartition(tableName, compactEffectivePredicate, buckets)), effectivePredicate, TupleDomain.none(), hiveBucketHandle);
    }
    List<Type> partitionTypes = partitionColumns.stream().map(column -> typeManager.getType(column.getTypeSignature())).collect(toList());
    List<String> partitionNames = getFilteredPartitionNames(metastore, tableName, partitionColumns, effectivePredicate);
    // do a final pass to filter based on fields that could not be used to filter the partitions
    int partitionCount = 0;
    ImmutableList.Builder<HivePartition> partitions = ImmutableList.builder();
    for (String partitionName : partitionNames) {
        Optional<Map<ColumnHandle, NullableValue>> values = parseValuesAndFilterPartition(partitionName, partitionColumns, partitionTypes, constraint);
        if (values.isPresent()) {
            if (partitionCount == maxPartitions) {
                throw new PrestoException(HIVE_EXCEEDED_PARTITION_LIMIT, format("Query over table '%s' can potentially read more than %s partitions", hiveTableHandle.getSchemaTableName(), maxPartitions));
            }
            partitionCount++;
            partitions.add(new HivePartition(tableName, compactEffectivePredicate, partitionName, values.get(), buckets));
        }
    }
    // All partition key domains will be fully evaluated, so we don't need to include those
    TupleDomain<ColumnHandle> remainingTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(effectivePredicate.getDomains().get(), not(Predicates.in(partitionColumns))));
    TupleDomain<ColumnHandle> enforcedTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(effectivePredicate.getDomains().get(), Predicates.in(partitionColumns)));
    return new HivePartitionResult(partitionColumns, partitions.build(), remainingTupleDomain, enforcedTupleDomain, hiveBucketHandle);
}

Example 2 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class HivePartitionManager method getFilteredPartitionNames.

private List<String> getFilteredPartitionNames(SemiTransactionalHiveMetastore metastore, SchemaTableName tableName, List<HiveColumnHandle> partitionKeys, TupleDomain<ColumnHandle> effectivePredicate) {
    checkArgument(effectivePredicate.getDomains().isPresent());
    List<String> filter = new ArrayList<>();
    for (HiveColumnHandle partitionKey : partitionKeys) {
        Domain domain = effectivePredicate.getDomains().get().get(partitionKey);
        if (domain != null && domain.isNullableSingleValue()) {
            Object value = domain.getNullableSingleValue();
            if (value == null) {
                filter.add(HivePartitionKey.HIVE_DEFAULT_DYNAMIC_PARTITION);
            } else if (value instanceof Slice) {
                filter.add(((Slice) value).toStringUtf8());
            } else if ((value instanceof Boolean) || (value instanceof Double) || (value instanceof Long)) {
                if (assumeCanonicalPartitionKeys) {
                    filter.add(value.toString());
                } else {
                    // Hive treats '0', 'false', and 'False' the same. However, the metastore differentiates between these.
                    filter.add(PARTITION_VALUE_WILDCARD);
                }
            } else {
                throw new PrestoException(NOT_SUPPORTED, "Only Boolean, Double and Long partition keys are supported");
            }
        } else {
            filter.add(PARTITION_VALUE_WILDCARD);
        }
    }
    // fetch the partition names
    return metastore.getPartitionNamesByParts(tableName.getSchemaName(), tableName.getTableName(), filter).orElseThrow(() -> new TableNotFoundException(tableName));
}

Example 3 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class HivePartitionManager method toCompactTupleDomain.

private static TupleDomain<HiveColumnHandle> toCompactTupleDomain(TupleDomain<ColumnHandle> effectivePredicate, int threshold) {
    checkArgument(effectivePredicate.getDomains().isPresent());
    ImmutableMap.Builder<HiveColumnHandle, Domain> builder = ImmutableMap.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) {
        HiveColumnHandle hiveColumnHandle = (HiveColumnHandle) entry.getKey();
        ValueSet values = entry.getValue().getValues();
        ValueSet compactValueSet = values.getValuesProcessor().<Optional<ValueSet>>transform(ranges -> ranges.getRangeCount() > threshold ? Optional.of(ValueSet.ofRanges(ranges.getSpan())) : Optional.empty(), discreteValues -> discreteValues.getValues().size() > threshold ? Optional.of(ValueSet.all(values.getType())) : Optional.empty(), allOrNone -> Optional.empty()).orElse(values);
        builder.put(hiveColumnHandle, Domain.create(compactValueSet, entry.getValue().isNullAllowed()));
    }
    return TupleDomain.withColumnDomains(builder.build());
}

Example 4 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class HiveBucketing method getHiveBucketNumbers.

public static List<HiveBucket> getHiveBucketNumbers(Table table, TupleDomain<ColumnHandle> effectivePredicate) {
    if (!table.getStorage().getBucketProperty().isPresent()) {
        return ImmutableList.of();
    }
    Optional<Map<ColumnHandle, NullableValue>> bindings = TupleDomain.extractFixedValues(effectivePredicate);
    if (!bindings.isPresent()) {
        return ImmutableList.of();
    }
    Optional<HiveBucket> singleBucket = getHiveBucket(table, bindings.get());
    if (singleBucket.isPresent()) {
        return ImmutableList.of(singleBucket.get());
    }
    if (!effectivePredicate.getDomains().isPresent()) {
        return ImmutableList.of();
    }
    Optional<Domain> domain = effectivePredicate.getDomains().get().entrySet().stream().filter(entry -> ((HiveColumnHandle) entry.getKey()).getName().equals(BUCKET_COLUMN_NAME)).findFirst().map(Entry::getValue);
    if (!domain.isPresent()) {
        return ImmutableList.of();
    }
    ValueSet values = domain.get().getValues();
    ImmutableList.Builder<HiveBucket> builder = ImmutableList.builder();
    int bucketCount = table.getStorage().getBucketProperty().get().getBucketCount();
    for (int i = 0; i < bucketCount; i++) {
        if (values.containsValue((long) i)) {
            builder.add(new HiveBucket(i, bucketCount));
        }
    }
    return builder.build();
}

Example 5 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class TestJdbcQueryBuilder method testNormalBuildSql.

@Test
public void testNormalBuildSql() throws SQLException {
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>builder().put(columns.get(0), Domain.create(SortedRangeSet.copyOf(BIGINT, ImmutableList.of(Range.equal(BIGINT, 128L), Range.equal(BIGINT, 180L), Range.equal(BIGINT, 233L), Range.lessThan(BIGINT, 25L), Range.range(BIGINT, 66L, true, 96L, true), Range.greaterThan(BIGINT, 192L))), false)).put(columns.get(1), Domain.create(SortedRangeSet.copyOf(DOUBLE, ImmutableList.of(Range.equal(DOUBLE, 200011.0), Range.equal(DOUBLE, 200014.0), Range.equal(DOUBLE, 200017.0), Range.equal(DOUBLE, 200116.5), Range.range(DOUBLE, 200030.0, true, 200036.0, true), Range.range(DOUBLE, 200048.0, true, 200099.0, true))), false)).put(columns.get(7), Domain.create(SortedRangeSet.copyOf(TINYINT, ImmutableList.of(Range.range(TINYINT, 60L, true, 70L, false), Range.range(TINYINT, 52L, true, 55L, false))), false)).put(columns.get(8), Domain.create(SortedRangeSet.copyOf(SMALLINT, ImmutableList.of(Range.range(SMALLINT, -75L, true, -68L, true), Range.range(SMALLINT, -200L, true, -100L, false))), false)).put(columns.get(9), Domain.create(SortedRangeSet.copyOf(INTEGER, ImmutableList.of(Range.equal(INTEGER, 80L), Range.equal(INTEGER, 96L), Range.lessThan(INTEGER, 0L))), false)).put(columns.get(2), Domain.create(SortedRangeSet.copyOf(BOOLEAN, ImmutableList.of(Range.equal(BOOLEAN, true))), false)).build());
    Connection connection = database.getConnection();
    try (PreparedStatement preparedStatement = new QueryBuilder("\"").buildSql(jdbcClient, connection, "", "", "test_table", columns, tupleDomain);
        ResultSet resultSet = preparedStatement.executeQuery()) {
        ImmutableSet.Builder<Long> builder = ImmutableSet.builder();
        while (resultSet.next()) {
            builder.add((Long) resultSet.getObject("col_0"));
        }
        assertEquals(builder.build(), ImmutableSet.of(68L, 180L, 196L));
    }
}