Examples with SortedRangeSet io.prestosql.spi.predicate.SortedRangeSet used on opensource projects

Example 1 with SortedRangeSet

use of io.prestosql.spi.predicate.SortedRangeSet in project hetu-core by openlookeng.

the class MemorySplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableHandle handle, SplitSchedulingStrategy splitSchedulingStrategy) {
    MemoryTableHandle table = (MemoryTableHandle) handle;
    List<MemoryDataFragment> dataFragments = metadata.getDataFragments(table.getId());
    // check if there is a predicate on the partition column
    List<SortedRangeSet> partitionKeyRanges = new ArrayList<>();
    for (Map.Entry<ColumnHandle, Domain> e : table.getPredicate().getDomains().orElse(Collections.emptyMap()).entrySet()) {
        if (!e.getKey().isPartitionKey()) {
            continue;
        }
        // see comment in LogicalPart#partitionPage also
        if (e.getValue().isNullAllowed()) {
            return allSplits(dataFragments, table);
        }
        SortedRangeSet rangeSet = ((SortedRangeSet) e.getValue().getValues());
        partitionKeyRanges.add(rangeSet);
    }
    ImmutableList.Builder<ConnectorSplit> splits = ImmutableList.builder();
    if (partitionKeyRanges.isEmpty()) {
        return allSplits(dataFragments, table);
    }
    for (MemoryDataFragment dataFragment : dataFragments) {
        Map<String, List<Integer>> logicalPartPartitionMap = dataFragment.getLogicalPartPartitionMap();
        long rows = dataFragment.getRows();
        if (logicalPartPartitionMap.size() == 0) {
            int logicalPartCount = dataFragment.getLogicalPartCount();
            // logicalPart ids are 1 based
            for (int i = 1; i <= logicalPartCount; i++) {
                splits.add(new MemorySplit(table.getId(), i, dataFragment.getHostAddress(), rows, OptionalLong.empty()));
            }
        } else {
            // filter the splits based on the partitionKey and only schedule them
            for (Map.Entry<String, List<Integer>> entry : logicalPartPartitionMap.entrySet()) {
                for (SortedRangeSet rangeSet : partitionKeyRanges) {
                    Type rangeSetType = rangeSet.getType();
                    Object value = LogicalPart.deserializeTypedValueFromString(rangeSetType, entry.getKey());
                    if (rangeSet.containsValue(value)) {
                        for (Integer i : entry.getValue()) {
                            splits.add(new MemorySplit(table.getId(), i, dataFragment.getHostAddress(), rows, OptionalLong.empty()));
                        }
                    }
                }
            }
        }
    }
    return new FixedSplitSource(splits.build());
}

Example 2 with SortedRangeSet

use of io.prestosql.spi.predicate.SortedRangeSet in project hetu-core by openlookeng.

the class ExpressionDomainTranslator method extractDisjuncts.

private List<Expression> extractDisjuncts(Type type, Ranges ranges, SymbolReference reference) {
    List<Expression> disjuncts = new ArrayList<>();
    List<Expression> singleValues = new ArrayList<>();
    List<Range> orderedRanges = ranges.getOrderedRanges();
    SortedRangeSet sortedRangeSet = SortedRangeSet.copyOf(type, orderedRanges);
    SortedRangeSet complement = sortedRangeSet.complement();
    List<Range> singleValueExclusionsList = complement.getOrderedRanges().stream().filter(Range::isSingleValue).collect(toList());
    List<Range> originalUnionSingleValues = SortedRangeSet.copyOf(type, singleValueExclusionsList).union(sortedRangeSet).getOrderedRanges();
    PeekingIterator<Range> singleValueExclusions = peekingIterator(singleValueExclusionsList.iterator());
    for (Range range : originalUnionSingleValues) {
        if (range.isSingleValue()) {
            singleValues.add(literalEncoder.toExpression(range.getSingleValue(), type));
            continue;
        }
        // attempt to optimize ranges that can be coalesced as long as single value points are excluded
        List<Expression> singleValuesInRange = new ArrayList<>();
        while (singleValueExclusions.hasNext() && range.contains(singleValueExclusions.peek())) {
            singleValuesInRange.add(literalEncoder.toExpression(singleValueExclusions.next().getSingleValue(), type));
        }
        if (!singleValuesInRange.isEmpty()) {
            disjuncts.add(combineRangeWithExcludedPoints(type, reference, range, singleValuesInRange));
            continue;
        }
        disjuncts.add(processRange(type, range, reference));
    }
    // Add back all of the possible single values either as an equality or an IN predicate
    if (singleValues.size() == 1) {
        disjuncts.add(new ComparisonExpression(EQUAL, reference, getOnlyElement(singleValues)));
    } else if (singleValues.size() > 1) {
        disjuncts.add(new InPredicate(reference, new InListExpression(singleValues)));
    }
    return disjuncts;
}

Example 3 with SortedRangeSet

use of io.prestosql.spi.predicate.SortedRangeSet in project hetu-core by openlookeng.

the class BitmapIndex method lookUp.

@Override
public Iterator<Integer> lookUp(Object expression) {
    checkClosed();
    if (expression instanceof Domain) {
        Domain predicate = (Domain) expression;
        List<Range> ranges = ((SortedRangeSet) (predicate.getValues())).getOrderedRanges();
        try {
            ArrayList<RoaringBitmap> allMatches = new ArrayList<>();
            for (Range range : ranges) {
                if (range.isSingleValue()) {
                    // unique value(for example: id=1, id in (1,2) (IN operator gives single exact values one by one)), bound: EXACTLY
                    Object value = getActualValue(predicate.getType(), range.getSingleValue());
                    Object byteArray = getBtreeReadOptimized().get(value);
                    if (byteArray != null) {
                        RoaringBitmap bitmap = byteArrayToBitmap(value, byteArray);
                        allMatches.add(bitmap);
                    }
                } else {
                    // <, <=, >=, >, BETWEEN
                    boolean highBoundless = range.getHigh().isUpperUnbounded();
                    boolean lowBoundless = range.getLow().isLowerUnbounded();
                    ConcurrentNavigableMap<Object, byte[]> concurrentNavigableMap = null;
                    if (highBoundless && !lowBoundless) {
                        Object low = getActualValue(predicate.getType(), range.getLow().getValue());
                        Object high = getBtreeReadOptimized().lastKey();
                        boolean fromInclusive = range.getLow().getBound().equals(Marker.Bound.EXACTLY);
                        if (getBtreeReadOptimized().comparator().compare(low, high) > 0) {
                            Object temp = low;
                            low = high;
                            high = temp;
                        }
                        concurrentNavigableMap = getBtreeReadOptimized().subMap(low, fromInclusive, high, true);
                    } else if (!highBoundless && lowBoundless) {
                        // <= or <
                        Object low = getBtreeReadOptimized().firstKey();
                        Object high = getActualValue(predicate.getType(), range.getHigh().getValue());
                        boolean toInclusive = range.getHigh().getBound().equals(Marker.Bound.EXACTLY);
                        if (getBtreeReadOptimized().comparator().compare(low, high) > 0) {
                            Object temp = low;
                            low = high;
                            high = temp;
                        }
                        concurrentNavigableMap = getBtreeReadOptimized().subMap(low, true, high, toInclusive);
                    } else if (!highBoundless && !lowBoundless) {
                        // BETWEEN
                        Object low = getActualValue(predicate.getType(), range.getLow().getValue());
                        Object high = getActualValue(predicate.getType(), range.getHigh().getValue());
                        if (getBtreeReadOptimized().comparator().compare(low, high) > 0) {
                            Object temp = low;
                            low = high;
                            high = temp;
                        }
                        concurrentNavigableMap = getBtreeReadOptimized().subMap(low, true, high, true);
                    } else {
                        // This case, combined gives a range of boundless for both high and low end
                        throw new UnsupportedOperationException("No use for bitmap index as all values are matched due to no bounds.");
                    }
                    for (Map.Entry<Object, byte[]> e : concurrentNavigableMap.entrySet()) {
                        if (e != null) {
                            RoaringBitmap bitmap = byteArrayToBitmap(e.getKey(), e.getValue());
                            allMatches.add(bitmap);
                        }
                    }
                }
            }
            if (allMatches.size() == 0) {
                return Collections.emptyIterator();
            }
            if (allMatches.size() == 1) {
                return allMatches.get(0).iterator();
            }
            return RoaringBitmap.or(allMatches.iterator()).iterator();
        } catch (Exception e) {
            throw new UnsupportedOperationException("Unsupported expression type.", e);
        }
    } else {
        throw new UnsupportedOperationException("Unsupported expression type.");
    }
}

Example 4 with SortedRangeSet

use of io.prestosql.spi.predicate.SortedRangeSet in project hetu-core by openlookeng.

the class TupleDomainFilterUtils method toFilter.

public static TupleDomainFilter toFilter(Domain domain) {
    ValueSet values = domain.getValues();
    checkArgument(values instanceof SortedRangeSet, "Unexpected domain type: " + values.getClass().getSimpleName());
    List<Range> ranges = ((SortedRangeSet) values).getOrderedRanges();
    boolean nullAllowed = domain.isNullAllowed();
    if (ranges.isEmpty() && nullAllowed) {
        return IS_NULL;
    }
    Type type = domain.getType();
    if (ranges.size() == 1 && type != BOOLEAN) {
        return createRangeFilter(type, ranges.get(0), nullAllowed);
    }
    if (type == BOOLEAN) {
        return createBooleanFilter(ranges, nullAllowed);
    }
    List<TupleDomainFilter> rangeFilters = ranges.stream().map(range -> createRangeFilter(type, range, false)).filter(not(ALWAYS_FALSE::equals)).collect(toImmutableList());
    if (rangeFilters.isEmpty()) {
        return nullAllowed ? IS_NULL : ALWAYS_FALSE;
    }
    if (rangeFilters.get(0) instanceof TupleDomainFilter.BigintRange) {
        List<TupleDomainFilter.BigintRange> bigintRanges = rangeFilters.stream().map(TupleDomainFilter.BigintRange.class::cast).collect(toImmutableList());
        // will get converted to BigIntValues so that filter will be evaluated on multiple possible values.
        if (bigintRanges.stream().allMatch(TupleDomainFilter.BigintRange::isSingleValue)) {
            return TupleDomainFilter.BigintValues.of(bigintRanges.stream().mapToLong(TupleDomainFilter.BigintRange::getLower).toArray(), nullAllowed);
        }
        // in any of these ranges.
        return TupleDomainFilter.BigintMultiRange.of(bigintRanges, nullAllowed);
    }
    return getMultiValuesTDF(rangeFilters, nullAllowed);
}

Example 5 with SortedRangeSet

use of io.prestosql.spi.predicate.SortedRangeSet in project hetu-core by openlookeng.

the class RowExpressionDomainTranslator method extractDisjuncts.

private List<RowExpression> extractDisjuncts(Type type, Ranges ranges, RowExpression reference) {
    List<RowExpression> disjuncts = new ArrayList<>();
    List<RowExpression> singleValues = new ArrayList<>();
    List<Range> orderedRanges = ranges.getOrderedRanges();
    SortedRangeSet sortedRangeSet = SortedRangeSet.copyOf(type, orderedRanges);
    SortedRangeSet complement = sortedRangeSet.complement();
    List<Range> singleValueExclusionsList = complement.getOrderedRanges().stream().filter(Range::isSingleValue).collect(toList());
    List<Range> originalUnionSingleValues = SortedRangeSet.copyOf(type, singleValueExclusionsList).union(sortedRangeSet).getOrderedRanges();
    PeekingIterator<Range> singleValueExclusions = peekingIterator(singleValueExclusionsList.iterator());
    for (Range range : originalUnionSingleValues) {
        if (range.isSingleValue()) {
            singleValues.add(toRowExpression(range.getSingleValue(), type));
            continue;
        }
        // attempt to optimize ranges that can be coalesced as long as single value points are excluded
        List<RowExpression> singleValuesInRange = new ArrayList<>();
        while (singleValueExclusions.hasNext() && range.contains(singleValueExclusions.peek())) {
            singleValuesInRange.add(toRowExpression(singleValueExclusions.next().getSingleValue(), type));
        }
        if (!singleValuesInRange.isEmpty()) {
            disjuncts.add(combineRangeWithExcludedPoints(type, reference, range, singleValuesInRange));
            continue;
        }
        disjuncts.add(processRange(type, range, reference));
    }
    // Add back all of the possible single values either as an equality or an IN predicate
    if (singleValues.size() == 1) {
        disjuncts.add(equal(reference, getOnlyElement(singleValues)));
    } else if (singleValues.size() > 1) {
        disjuncts.add(in(reference, singleValues));
    }
    return disjuncts;
}