Examples with TupleDomain com.facebook.presto.spi.predicate.TupleDomain used on opensource projects

Example 21 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class AbstractTestHiveClient method doTestMetadataDelete.

private void doTestMetadataDelete(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception {
    // creating the table
    doCreateEmptyTable(tableName, storageFormat, CREATE_TABLE_COLUMNS_PARTITIONED);
    insertData(tableName, CREATE_TABLE_PARTITIONED_DATA);
    MaterializedResult.Builder expectedResultBuilder = MaterializedResult.resultBuilder(SESSION, CREATE_TABLE_PARTITIONED_DATA.getTypes());
    expectedResultBuilder.rows(CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows());
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        // verify partitions were created
        List<String> partitionNames = transaction.getMetastore(tableName.getSchemaName()).getPartitionNames(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new PrestoException(HIVE_METASTORE_ERROR, "Partition metadata not available"));
        assertEqualsIgnoreOrder(partitionNames, CREATE_TABLE_PARTITIONED_DATA.getMaterializedRows().stream().map(row -> "ds=" + row.getField(CREATE_TABLE_PARTITIONED_DATA.getTypes().size() - 1)).collect(toList()));
        // verify table directory is not empty
        Set<String> filesAfterInsert = listAllDataFiles(transaction, tableName.getSchemaName(), tableName.getTableName());
        assertFalse(filesAfterInsert.isEmpty());
        // verify the data
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
        assertEqualsIgnoreOrder(result.getMaterializedRows(), expectedResultBuilder.build().getMaterializedRows());
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        // get ds column handle
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle).get("ds");
        // delete ds=2015-07-03
        session = newSession();
        TupleDomain<ColumnHandle> tupleDomain = TupleDomain.fromFixedValues(ImmutableMap.of(dsColumnHandle, NullableValue.of(createUnboundedVarcharType(), utf8Slice("2015-07-03"))));
        Constraint<ColumnHandle> constraint = new Constraint<>(tupleDomain, convertToPredicate(tupleDomain));
        List<ConnectorTableLayoutResult> tableLayoutResults = metadata.getTableLayouts(session, tableHandle, constraint, Optional.empty());
        ConnectorTableLayoutHandle tableLayoutHandle = Iterables.getOnlyElement(tableLayoutResults).getTableLayout().getHandle();
        metadata.metadataDelete(session, tableHandle, tableLayoutHandle);
        transaction.commit();
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = filterNonHiddenColumnHandles(metadata.getColumnHandles(session, tableHandle).values());
        HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle).get("ds");
        int dsColumnOrdinalPosition = columnHandles.indexOf(dsColumnHandle);
        // verify the data
        session = newSession();
        ImmutableList<MaterializedRow> expectedRows = expectedResultBuilder.build().getMaterializedRows().stream().filter(row -> !"2015-07-03".equals(row.getField(dsColumnOrdinalPosition))).collect(ImmutableCollectors.toImmutableList());
        MaterializedResult actualAfterDelete = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
        assertEqualsIgnoreOrder(actualAfterDelete.getMaterializedRows(), expectedRows);
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        HiveColumnHandle dsColumnHandle = (HiveColumnHandle) metadata.getColumnHandles(session, tableHandle).get("ds");
        // delete ds=2015-07-01 and 2015-07-02
        session = newSession();
        TupleDomain<ColumnHandle> tupleDomain2 = TupleDomain.withColumnDomains(ImmutableMap.of(dsColumnHandle, Domain.create(ValueSet.ofRanges(Range.range(createUnboundedVarcharType(), utf8Slice("2015-07-01"), true, utf8Slice("2015-07-02"), true)), false)));
        Constraint<ColumnHandle> constraint2 = new Constraint<>(tupleDomain2, convertToPredicate(tupleDomain2));
        List<ConnectorTableLayoutResult> tableLayoutResults2 = metadata.getTableLayouts(session, tableHandle, constraint2, Optional.empty());
        ConnectorTableLayoutHandle tableLayoutHandle2 = Iterables.getOnlyElement(tableLayoutResults2).getTableLayout().getHandle();
        metadata.metadataDelete(session, tableHandle, tableLayoutHandle2);
        transaction.commit();
    }
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        List<ColumnHandle> columnHandles = ImmutableList.copyOf(metadata.getColumnHandles(session, tableHandle).values());
        // verify the data
        session = newSession();
        MaterializedResult actualAfterDelete2 = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.of(storageFormat));
        assertEqualsIgnoreOrder(actualAfterDelete2.getMaterializedRows(), ImmutableList.of());
        // verify table directory is empty
        Set<String> filesAfterDelete = listAllDataFiles(transaction, tableName.getSchemaName(), tableName.getTableName());
        assertTrue(filesAfterDelete.isEmpty());
    }
}

Example 22 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project carbondata by apache.

the class CarbondataRecordSetProvider method fillFilter2QueryModel.

// Build filter for QueryModel
private void fillFilter2QueryModel(QueryModel queryModel, TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) {
    //queryModel.setFilterExpressionResolverTree(new FilterResolverIntf());
    //Build Predicate Expression
    ImmutableList.Builder<Expression> filters = ImmutableList.builder();
    Domain domain = null;
    for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
        // Build ColumnExpresstion for Expresstion(Carbondata)
        CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
        Type type = cdch.getColumnType();
        DataType coltype = Spi2CarbondataTypeMapper(cdch);
        Expression colExpression = new ColumnExpression(cdch.getColumnName(), coltype);
        domain = originalConstraint.getDomains().get().get(c);
        checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
        if (domain.getValues().isNone()) {
        }
        if (domain.getValues().isAll()) {
        }
        List<Object> singleValues = new ArrayList<>();
        List<Expression> rangeFilter = new ArrayList<>();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            // Already checked
            checkState(!range.isAll());
            if (range.isSingleValue()) {
                singleValues.add(range.getLow().getValue());
            } else {
                List<String> rangeConjuncts = new ArrayList<>();
                if (!range.getLow().isLowerUnbounded()) {
                    Object value = ConvertDataByType(range.getLow().getValue(), type);
                    switch(range.getLow().getBound()) {
                        case ABOVE:
                            if (type == TimestampType.TIMESTAMP) {
                            //todo not now
                            } else {
                                GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
                                rangeFilter.add(greater);
                            }
                            break;
                        case EXACTLY:
                            GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(greater);
                            break;
                        case BELOW:
                            throw new IllegalArgumentException("Low marker should never use BELOW bound");
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
                    }
                }
                if (!range.getHigh().isUpperUnbounded()) {
                    Object value = ConvertDataByType(range.getHigh().getValue(), type);
                    switch(range.getHigh().getBound()) {
                        case ABOVE:
                            throw new IllegalArgumentException("High marker should never use ABOVE bound");
                        case EXACTLY:
                            LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less);
                            break;
                        case BELOW:
                            LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less2);
                            break;
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
                    }
                }
            }
        }
        if (singleValues.size() == 1) {
            Expression ex = null;
            if (coltype.equals(DataType.STRING)) {
                ex = new EqualToExpression(colExpression, new LiteralExpression(((Slice) singleValues.get(0)).toStringUtf8(), coltype));
            } else if (coltype.equals(DataType.TIMESTAMP) || coltype.equals(DataType.DATE)) {
                Long value = (Long) singleValues.get(0) * 1000;
                ex = new EqualToExpression(colExpression, new LiteralExpression(value, coltype));
            } else
                ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
            filters.add(ex);
        } else if (singleValues.size() > 1) {
            ListExpression candidates = null;
            List<Expression> exs = singleValues.stream().map((a) -> {
                return new LiteralExpression(ConvertDataByType(a, type), coltype);
            }).collect(Collectors.toList());
            candidates = new ListExpression(exs);
            if (candidates != null)
                filters.add(new InExpression(colExpression, candidates));
        } else if (rangeFilter.size() > 0) {
            if (rangeFilter.size() > 1) {
                Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
                if (rangeFilter.size() > 2) {
                    for (int i = 2; i < rangeFilter.size(); i++) {
                        filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
                    }
                }
            } else if (rangeFilter.size() == 1)
                filters.add(rangeFilter.get(0));
        }
    }
    Expression finalFilters;
    List<Expression> tmp = filters.build();
    if (tmp.size() > 1) {
        finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
        if (tmp.size() > 2) {
            for (int i = 2; i < tmp.size(); i++) {
                finalFilters = new AndExpression(finalFilters, tmp.get(i));
            }
        }
    } else if (tmp.size() == 1)
        finalFilters = tmp.get(0);
    else
        return;
    // todo set into QueryModel
    CarbonInputFormatUtil.processFilterExpression(finalFilters, carbonTable);
    queryModel.setFilterExpressionResolverTree(CarbonInputFormatUtil.resolveFilter(finalFilters, queryModel.getAbsoluteTableIdentifier()));
}

Example 23 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project carbondata by apache.

the class CarbondataSplitManager method parseFilterExpression.

/**
   * Convert presto-TupleDomain predication into Carbon scan express condition
   * @param originalConstraint  presto-TupleDomain
   * @param carbonTable
   * @return
   */
public Expression parseFilterExpression(TupleDomain<ColumnHandle> originalConstraint, CarbonTable carbonTable) {
    ImmutableList.Builder<Expression> filters = ImmutableList.builder();
    Domain domain = null;
    for (ColumnHandle c : originalConstraint.getDomains().get().keySet()) {
        CarbondataColumnHandle cdch = (CarbondataColumnHandle) c;
        Type type = cdch.getColumnType();
        List<CarbonColumn> ccols = carbonTable.getCreateOrderColumn(carbonTable.getFactTableName());
        Optional<CarbonColumn> target = ccols.stream().filter(a -> a.getColName().equals(cdch.getColumnName())).findFirst();
        if (target.get() == null)
            return null;
        DataType coltype = target.get().getDataType();
        ColumnExpression colExpression = new ColumnExpression(cdch.getColumnName(), target.get().getDataType());
        //colExpression.setColIndex(cs.getSchemaOrdinal());
        colExpression.setDimension(target.get().isDimension());
        colExpression.setDimension(carbonTable.getDimensionByName(carbonTable.getFactTableName(), cdch.getColumnName()));
        colExpression.setCarbonColumn(target.get());
        domain = originalConstraint.getDomains().get().get(c);
        checkArgument(domain.getType().isOrderable(), "Domain type must be orderable");
        if (domain.getValues().isNone()) {
        }
        if (domain.getValues().isAll()) {
        }
        List<Object> singleValues = new ArrayList<>();
        List<Expression> rangeFilter = new ArrayList<>();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            // Already checked
            checkState(!range.isAll());
            if (range.isSingleValue()) {
                singleValues.add(range.getLow().getValue());
            } else {
                List<String> rangeConjuncts = new ArrayList<>();
                if (!range.getLow().isLowerUnbounded()) {
                    Object value = ConvertDataByType(range.getLow().getValue(), type);
                    switch(range.getLow().getBound()) {
                        case ABOVE:
                            if (type == TimestampType.TIMESTAMP) {
                            //todo not now
                            } else {
                                GreaterThanExpression greater = new GreaterThanExpression(colExpression, new LiteralExpression(value, coltype));
                                rangeFilter.add(greater);
                            }
                            break;
                        case EXACTLY:
                            GreaterThanEqualToExpression greater = new GreaterThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(greater);
                            break;
                        case BELOW:
                            throw new IllegalArgumentException("Low marker should never use BELOW bound");
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getLow().getBound());
                    }
                }
                if (!range.getHigh().isUpperUnbounded()) {
                    Object value = ConvertDataByType(range.getHigh().getValue(), type);
                    switch(range.getHigh().getBound()) {
                        case ABOVE:
                            throw new IllegalArgumentException("High marker should never use ABOVE bound");
                        case EXACTLY:
                            LessThanEqualToExpression less = new LessThanEqualToExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less);
                            break;
                        case BELOW:
                            LessThanExpression less2 = new LessThanExpression(colExpression, new LiteralExpression(value, coltype));
                            rangeFilter.add(less2);
                            break;
                        default:
                            throw new AssertionError("Unhandled bound: " + range.getHigh().getBound());
                    }
                }
            }
        }
        if (singleValues.size() == 1) {
            Expression ex = null;
            if (coltype.equals(DataType.STRING)) {
                ex = new EqualToExpression(colExpression, new LiteralExpression(((Slice) singleValues.get(0)).toStringUtf8(), coltype));
            } else
                ex = new EqualToExpression(colExpression, new LiteralExpression(singleValues.get(0), coltype));
            filters.add(ex);
        } else if (singleValues.size() > 1) {
            ListExpression candidates = null;
            List<Expression> exs = singleValues.stream().map((a) -> {
                return new LiteralExpression(ConvertDataByType(a, type), coltype);
            }).collect(Collectors.toList());
            candidates = new ListExpression(exs);
            if (candidates != null)
                filters.add(new InExpression(colExpression, candidates));
        } else if (rangeFilter.size() > 0) {
            if (rangeFilter.size() > 1) {
                Expression finalFilters = new OrExpression(rangeFilter.get(0), rangeFilter.get(1));
                if (rangeFilter.size() > 2) {
                    for (int i = 2; i < rangeFilter.size(); i++) {
                        filters.add(new AndExpression(finalFilters, rangeFilter.get(i)));
                    }
                }
            } else if (//only have one value
            rangeFilter.size() == 1)
                filters.add(rangeFilter.get(0));
        }
    }
    Expression finalFilters;
    List<Expression> tmp = filters.build();
    if (tmp.size() > 1) {
        finalFilters = new AndExpression(tmp.get(0), tmp.get(1));
        if (tmp.size() > 2) {
            for (int i = 2; i < tmp.size(); i++) {
                finalFilters = new AndExpression(finalFilters, tmp.get(i));
            }
        }
    } else if (tmp.size() == 1)
        finalFilters = tmp.get(0);
    else
        //no filter
        return null;
    return finalFilters;
}

Example 24 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class CassandraClusteringPredicatesExtractor method getClusteringKeysSet.

private static ClusteringPushDownResult getClusteringKeysSet(List<CassandraColumnHandle> clusteringColumns, TupleDomain<ColumnHandle> predicates) {
    ImmutableMap.Builder<ColumnHandle, Domain> domainsBuilder = ImmutableMap.builder();
    ImmutableList.Builder<Set<Object>> clusteringColumnValues = ImmutableList.builder();
    for (CassandraColumnHandle columnHandle : clusteringColumns) {
        Domain domain = predicates.getDomains().get().get(columnHandle);
        if (domain == null) {
            break;
        }
        if (domain.isNullAllowed()) {
            return new ClusteringPushDownResult(domainsBuilder.build(), ImmutableSet.of());
        }
        Set<Object> values = domain.getValues().getValuesProcessor().transform(ranges -> {
            ImmutableSet.Builder<Object> columnValues = ImmutableSet.builder();
            for (Range range : ranges.getOrderedRanges()) {
                if (!range.isSingleValue()) {
                    return ImmutableSet.of();
                }
                Object value = range.getSingleValue();
                CassandraType valueType = columnHandle.getCassandraType();
                columnValues.add(valueType.validateClusteringKey(value));
            }
            return columnValues.build();
        }, discreteValues -> {
            if (discreteValues.isWhiteList()) {
                return ImmutableSet.copyOf(discreteValues.getValues());
            }
            return ImmutableSet.of();
        }, allOrNone -> ImmutableSet.of());
        if (!values.isEmpty()) {
            clusteringColumnValues.add(values);
            domainsBuilder.put(columnHandle, domain);
        }
    }
    return new ClusteringPushDownResult(domainsBuilder.build(), cartesianProduct(clusteringColumnValues.build()));
}

Example 25 with TupleDomain

use of com.facebook.presto.spi.predicate.TupleDomain in project presto by prestodb.

the class CassandraPartitionManager method getPartitions.

public CassandraPartitionResult getPartitions(ConnectorTableHandle tableHandle, TupleDomain<ColumnHandle> tupleDomain) {
    CassandraTableHandle cassandraTableHandle = (CassandraTableHandle) tableHandle;
    CassandraTable table = schemaProvider.getTable(cassandraTableHandle);
    List<CassandraColumnHandle> partitionKeys = table.getPartitionKeyColumns();
    // fetch the partitions
    List<CassandraPartition> allPartitions = getCassandraPartitions(table, tupleDomain);
    log.debug("%s.%s #partitions: %d", cassandraTableHandle.getSchemaName(), cassandraTableHandle.getTableName(), allPartitions.size());
    // do a final pass to filter based on fields that could not be used to build the prefix
    List<CassandraPartition> partitions = allPartitions.stream().filter(partition -> tupleDomain.overlaps(partition.getTupleDomain())).collect(toList());
    // All partition key domains will be fully evaluated, so we don't need to include those
    TupleDomain<ColumnHandle> remainingTupleDomain = TupleDomain.none();
    if (!tupleDomain.isNone()) {
        if (partitions.size() == 1 && partitions.get(0).isUnpartitioned()) {
            remainingTupleDomain = tupleDomain;
        } else {
            @SuppressWarnings({ "rawtypes", "unchecked" }) List<ColumnHandle> partitionColumns = (List) partitionKeys;
            remainingTupleDomain = TupleDomain.withColumnDomains(Maps.filterKeys(tupleDomain.getDomains().get(), not(in(partitionColumns))));
        }
    }
    // push down indexed column fixed value predicates only for unpartitioned partition which uses token range query
    if ((partitions.size() == 1) && partitions.get(0).isUnpartitioned()) {
        Map<ColumnHandle, Domain> domains = tupleDomain.getDomains().get();
        List<ColumnHandle> indexedColumns = new ArrayList<>();
        // compose partitionId by using indexed column
        StringBuilder sb = new StringBuilder();
        for (Map.Entry<ColumnHandle, Domain> entry : domains.entrySet()) {
            CassandraColumnHandle column = (CassandraColumnHandle) entry.getKey();
            Domain domain = entry.getValue();
            if (column.isIndexed() && domain.isSingleValue()) {
                sb.append(CassandraCqlUtils.validColumnName(column.getName())).append(" = ").append(CassandraCqlUtils.cqlValue(toCQLCompatibleString(entry.getValue().getSingleValue()), column.getCassandraType()));
                indexedColumns.add(column);
                // Only one indexed column predicate can be pushed down.
                break;
            }
        }
        if (sb.length() > 0) {
            CassandraPartition partition = partitions.get(0);
            TupleDomain<ColumnHandle> filterIndexedColumn = TupleDomain.withColumnDomains(Maps.filterKeys(remainingTupleDomain.getDomains().get(), not(in(indexedColumns))));
            partitions = new ArrayList<>();
            partitions.add(new CassandraPartition(partition.getKey(), sb.toString(), filterIndexedColumn, true));
            return new CassandraPartitionResult(partitions, filterIndexedColumn);
        }
    }
    return new CassandraPartitionResult(partitions, remainingTupleDomain);
}