Examples with TupleDomain io.trino.spi.predicate.TupleDomain used on opensource projects

Example 21 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class TestLocalDynamicFilterConsumer method testMultiplePartitions.

@Test
public void testMultiplePartitions() throws Exception {
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(ImmutableMap.of(new DynamicFilterId("123"), 0), ImmutableMap.of(new DynamicFilterId("123"), INTEGER), 2);
    assertEquals(filter.getBuildChannels(), ImmutableMap.of(new DynamicFilterId("123"), 0));
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<DynamicFilterId, Domain>> result = filter.getDynamicFilterDomains();
    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.singleValue(INTEGER, 10L))));
    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(new DynamicFilterId("123"), Domain.singleValue(INTEGER, 20L))));
    assertEquals(result.get(), ImmutableMap.of(new DynamicFilterId("123"), Domain.multipleValues(INTEGER, ImmutableList.of(10L, 20L))));
}

Example 22 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class CassandraClusteringPredicatesExtractor method getClusteringKeysSet.

private static ClusteringPushDownResult getClusteringKeysSet(List<CassandraColumnHandle> clusteringColumns, TupleDomain<ColumnHandle> predicates, VersionNumber cassandraVersion) {
    ImmutableSet.Builder<ColumnHandle> fullyPushedColumnPredicates = ImmutableSet.builder();
    ImmutableList.Builder<String> clusteringColumnSql = ImmutableList.builder();
    int allProcessedClusteringColumns = 0;
    for (CassandraColumnHandle columnHandle : clusteringColumns) {
        Domain domain = predicates.getDomains().get().get(columnHandle);
        if (domain == null) {
            break;
        }
        if (domain.isNullAllowed()) {
            break;
        }
        int currentlyProcessedClusteringColumn = allProcessedClusteringColumns;
        String predicateString = domain.getValues().getValuesProcessor().transform(ranges -> {
            if (ranges.getRangeCount() == 1) {
                fullyPushedColumnPredicates.add(columnHandle);
                return translateRangeIntoCql(columnHandle, getOnlyElement(ranges.getOrderedRanges()));
            }
            if (ranges.getOrderedRanges().stream().allMatch(Range::isSingleValue)) {
                if (isInExpressionNotAllowed(clusteringColumns, cassandraVersion, currentlyProcessedClusteringColumn)) {
                    return translateRangeIntoCql(columnHandle, ranges.getSpan());
                }
                String inValues = ranges.getOrderedRanges().stream().map(range -> toCqlLiteral(columnHandle, range.getSingleValue())).collect(joining(","));
                fullyPushedColumnPredicates.add(columnHandle);
                return CassandraCqlUtils.validColumnName(columnHandle.getName()) + " IN (" + inValues + ")";
            }
            return translateRangeIntoCql(columnHandle, ranges.getSpan());
        }, discreteValues -> {
            if (discreteValues.isInclusive()) {
                if (discreteValues.getValuesCount() == 0) {
                    return null;
                }
                if (discreteValues.getValuesCount() == 1) {
                    fullyPushedColumnPredicates.add(columnHandle);
                    return format("%s = %s", CassandraCqlUtils.validColumnName(columnHandle.getName()), toCqlLiteral(columnHandle, getOnlyElement(discreteValues.getValues())));
                }
                if (isInExpressionNotAllowed(clusteringColumns, cassandraVersion, currentlyProcessedClusteringColumn)) {
                    return null;
                }
                String inValues = discreteValues.getValues().stream().map(columnHandle.getCassandraType()::toCqlLiteral).collect(joining(","));
                fullyPushedColumnPredicates.add(columnHandle);
                return CassandraCqlUtils.validColumnName(columnHandle.getName()) + " IN (" + inValues + " )";
            }
            return null;
        }, allOrNone -> null);
        if (predicateString == null) {
            break;
        }
        clusteringColumnSql.add(predicateString);
        // Check for last clustering column should only be restricted by range condition
        if (predicateString.contains(">") || predicateString.contains("<")) {
            break;
        }
        allProcessedClusteringColumns++;
    }
    List<String> clusteringColumnPredicates = clusteringColumnSql.build();
    return new ClusteringPushDownResult(fullyPushedColumnPredicates.build(), Joiner.on(" AND ").join(clusteringColumnPredicates));
}

Example 23 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class CassandraPartitionManager method getPartitions.

public CassandraPartitionResult getPartitions(CassandraTableHandle cassandraTableHandle, TupleDomain<ColumnHandle> tupleDomain) {
    // TODO support repeated applyFilter
    checkArgument(cassandraTableHandle.getPartitions().isEmpty(), "getPartitions() currently does not take into account table handle's partitions");
    CassandraTable table = cassandraSession.getTable(cassandraTableHandle.getSchemaTableName());
    // fetch the partitions
    List<CassandraPartition> allPartitions = getCassandraPartitions(table, tupleDomain);
    log.debug("%s.%s #partitions: %d", cassandraTableHandle.getSchemaName(), cassandraTableHandle.getTableName(), allPartitions.size());
    // do a final pass to filter based on fields that could not be used to build the prefix
    List<CassandraPartition> partitions = allPartitions.stream().filter(partition -> tupleDomain.overlaps(partition.getTupleDomain())).collect(toList());
    // All partition key domains will be fully evaluated, so we don't need to include those
    TupleDomain<ColumnHandle> remainingTupleDomain = TupleDomain.none();
    if (!tupleDomain.isNone()) {
        if (partitions.size() == 1 && partitions.get(0).isUnpartitioned()) {
            remainingTupleDomain = tupleDomain;
        } else {
            Set<ColumnHandle> usedPartitionColumns = partitions.stream().flatMap(partition -> Optional.ofNullable(partition.getTupleDomain()).flatMap(partitionTupleDomain -> partitionTupleDomain.getDomains().map(Map::keySet).map(Set::stream)).orElse(Stream.empty())).collect(toImmutableSet());
            remainingTupleDomain = tupleDomain.filter((column, domain) -> !usedPartitionColumns.contains(column));
        }
    }
    // push down indexed column fixed value predicates only for unpartitioned partition which uses token range query
    if ((partitions.size() == 1) && partitions.get(0).isUnpartitioned()) {
        Map<ColumnHandle, Domain> domains = tupleDomain.getDomains().get();
        List<ColumnHandle> indexedColumns = new ArrayList<>();
        // compose partitionId by using indexed column
        StringBuilder sb = new StringBuilder();
        for (Map.Entry<ColumnHandle, Domain> entry : domains.entrySet()) {
            CassandraColumnHandle column = (CassandraColumnHandle) entry.getKey();
            Domain domain = entry.getValue();
            if (column.isIndexed() && domain.isSingleValue()) {
                sb.append(CassandraCqlUtils.validColumnName(column.getName())).append(" = ").append(column.getCassandraType().toCqlLiteral(entry.getValue().getSingleValue()));
                indexedColumns.add(column);
                // Only one indexed column predicate can be pushed down.
                break;
            }
        }
        if (sb.length() > 0) {
            CassandraPartition partition = partitions.get(0);
            TupleDomain<ColumnHandle> filterIndexedColumn = TupleDomain.withColumnDomains(Maps.filterKeys(remainingTupleDomain.getDomains().get(), not(in(indexedColumns))));
            partitions = new ArrayList<>();
            partitions.add(new CassandraPartition(partition.getKey(), sb.toString(), filterIndexedColumn, true));
            return new CassandraPartitionResult(partitions, filterIndexedColumn);
        }
    }
    return new CassandraPartitionResult(partitions, remainingTupleDomain);
}

Example 24 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class LocalDynamicFiltersCollector method createDynamicFilter.

// Called during TableScan planning (no need to be synchronized as local planning is single threaded)
public DynamicFilter createDynamicFilter(List<Descriptor> descriptors, Map<Symbol, ColumnHandle> columnsMap, TypeProvider typeProvider, PlannerContext plannerContext) {
    Multimap<DynamicFilterId, Descriptor> descriptorMap = extractSourceSymbols(descriptors);
    // Iterate over dynamic filters that are collected (correspond to one of the futures), and required for filtering (correspond to one of the descriptors).
    // It is possible that some dynamic filters are collected in a different stage - and will not available here.
    // It is also possible that not all local dynamic filters are needed for this specific table scan.
    List<ListenableFuture<TupleDomain<ColumnHandle>>> predicateFutures = descriptorMap.keySet().stream().filter(futures.keySet()::contains).map(filterId -> {
        // Probe-side columns that can be filtered with this dynamic filter resulting domain.
        return Futures.transform(requireNonNull(futures.get(filterId), () -> format("Missing dynamic filter %s", filterId)), // Construct a probe-side predicate by duplicating the resulting domain over the corresponding columns.
        domain -> TupleDomain.withColumnDomains(descriptorMap.get(filterId).stream().collect(toImmutableMap(descriptor -> {
            Symbol probeSymbol = Symbol.from(descriptor.getInput());
            return requireNonNull(columnsMap.get(probeSymbol), () -> format("Missing probe column for %s", probeSymbol));
        }, descriptor -> {
            Type targetType = typeProvider.get(Symbol.from(descriptor.getInput()));
            Domain updatedDomain = descriptor.applyComparison(domain);
            if (!updatedDomain.getType().equals(targetType)) {
                return applySaturatedCasts(plannerContext.getMetadata(), plannerContext.getFunctionManager(), plannerContext.getTypeOperators(), session, updatedDomain, targetType);
            }
            return updatedDomain;
        }))), directExecutor());
    }).collect(toImmutableList());
    Set<ColumnHandle> columnsCovered = descriptorMap.values().stream().map(Descriptor::getInput).map(Symbol::from).map(probeSymbol -> requireNonNull(columnsMap.get(probeSymbol), () -> "Missing probe column for " + probeSymbol)).collect(toImmutableSet());
    return new TableSpecificDynamicFilter(columnsCovered, predicateFutures);
}

Example 25 with TupleDomain

use of io.trino.spi.predicate.TupleDomain in project trino by trinodb.

the class TestHiveGlueMetastore method doGetPartitionsFilterTest.

/**
 * @param filterList should be same sized list as expectedValuesList
 * @param expectedValuesList
 * @throws Exception
 */
private void doGetPartitionsFilterTest(List<ColumnMetadata> columnMetadata, List<String> partitionColumnNames, List<PartitionValues> partitionValues, List<TupleDomain<String>> filterList, List<List<PartitionValues>> expectedValuesList) throws Exception {
    try (CloseableSchamaTableName closeableTableName = new CloseableSchamaTableName(temporaryTable(("get_partitions")))) {
        SchemaTableName tableName = closeableTableName.getSchemaTableName();
        createDummyPartitionedTable(tableName, columnMetadata, partitionColumnNames, partitionValues);
        HiveMetastore metastoreClient = getMetastoreClient();
        for (int i = 0; i < filterList.size(); i++) {
            TupleDomain<String> filter = filterList.get(i);
            List<PartitionValues> expectedValues = expectedValuesList.get(i);
            List<String> expectedResults = expectedValues.stream().map(expectedPartitionValues -> makePartName(partitionColumnNames, expectedPartitionValues.getValues())).collect(toImmutableList());
            Optional<List<String>> partitionNames = metastoreClient.getPartitionNamesByFilter(tableName.getSchemaName(), tableName.getTableName(), partitionColumnNames, filter);
            assertTrue(partitionNames.isPresent());
            assertEquals(partitionNames.get(), expectedResults, format("lists \nactual: %s\nexpected: %s\nmismatch for filter %s (input index %d)\n", partitionNames.get(), expectedResults, filter, i));
        }
    }
}