Examples with StringComparator org.apache.druid.query.ordering.StringComparator used on opensource projects

Example 6 with StringComparator

use of org.apache.druid.query.ordering.StringComparator in project druid by druid-io.

the class GrouperBufferComparatorUtils method bufferComparatorWithAggregators.

public static Grouper.BufferComparator bufferComparatorWithAggregators(AggregatorFactory[] aggregatorFactories, int[] aggregatorOffsets, DefaultLimitSpec limitSpec, List<DimensionSpec> dimensions, Grouper.BufferComparator[] dimComparators, boolean includeTimestamp, boolean sortByDimsFirst, int keySize) {
    int dimCount = dimensions.size();
    final List<Boolean> needsReverses = new ArrayList<>();
    List<Grouper.BufferComparator> comparators = new ArrayList<>();
    Set<Integer> orderByIndices = new HashSet<>();
    int aggCount = 0;
    boolean needsReverse;
    for (OrderByColumnSpec orderSpec : limitSpec.getColumns()) {
        needsReverse = orderSpec.getDirection() != OrderByColumnSpec.Direction.ASCENDING;
        int dimIndex = OrderByColumnSpec.getDimIndexForOrderBy(orderSpec, dimensions);
        if (dimIndex >= 0) {
            comparators.add(dimComparators[dimIndex]);
            orderByIndices.add(dimIndex);
            needsReverses.add(needsReverse);
        } else {
            int aggIndex = OrderByColumnSpec.getAggIndexForOrderBy(orderSpec, Arrays.asList(aggregatorFactories));
            if (aggIndex >= 0) {
                final StringComparator stringComparator = orderSpec.getDimensionComparator();
                final ColumnType valueType = aggregatorFactories[aggIndex].getIntermediateType();
                // Aggregators start after dimensions
                final int aggOffset = keySize + aggregatorOffsets[aggIndex];
                aggCount++;
                if (!valueType.isNumeric()) {
                    throw new IAE("Cannot order by a non-numeric aggregator[%s]", orderSpec);
                }
                comparators.add(makeNullHandlingBufferComparatorForNumericData(aggOffset, makeNumericBufferComparator(valueType, aggOffset, true, stringComparator)));
                needsReverses.add(needsReverse);
            }
        }
    }
    for (int i = 0; i < dimCount; i++) {
        if (!orderByIndices.contains(i)) {
            comparators.add(dimComparators[i]);
            // default to Ascending order if dim is not in an orderby spec
            needsReverses.add(false);
        }
    }
    final Grouper.BufferComparator[] adjustedSerdeHelperComparators = comparators.toArray(new Grouper.BufferComparator[0]);
    final int fieldCount = dimCount + aggCount;
    if (includeTimestamp) {
        if (sortByDimsFirst) {
            return new Grouper.BufferComparator() {

                @Override
                public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) {
                    final int cmp = compareDimsInBuffersForNullFudgeTimestampForPushDown(adjustedSerdeHelperComparators, needsReverses, fieldCount, lhsBuffer, rhsBuffer, lhsPosition, rhsPosition);
                    if (cmp != 0) {
                        return cmp;
                    }
                    return Longs.compare(lhsBuffer.getLong(lhsPosition), rhsBuffer.getLong(rhsPosition));
                }
            };
        } else {
            return new Grouper.BufferComparator() {

                @Override
                public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) {
                    final int timeCompare = Longs.compare(lhsBuffer.getLong(lhsPosition), rhsBuffer.getLong(rhsPosition));
                    if (timeCompare != 0) {
                        return timeCompare;
                    }
                    int cmp = compareDimsInBuffersForNullFudgeTimestampForPushDown(adjustedSerdeHelperComparators, needsReverses, fieldCount, lhsBuffer, rhsBuffer, lhsPosition, rhsPosition);
                    return cmp;
                }
            };
        }
    } else {
        return new Grouper.BufferComparator() {

            @Override
            public int compare(ByteBuffer lhsBuffer, ByteBuffer rhsBuffer, int lhsPosition, int rhsPosition) {
                for (int i = 0; i < fieldCount; i++) {
                    final int cmp;
                    if (needsReverses.get(i)) {
                        cmp = adjustedSerdeHelperComparators[i].compare(rhsBuffer, lhsBuffer, rhsPosition, lhsPosition);
                    } else {
                        cmp = adjustedSerdeHelperComparators[i].compare(lhsBuffer, rhsBuffer, lhsPosition, rhsPosition);
                    }
                    if (cmp != 0) {
                        return cmp;
                    }
                }
                return 0;
            }
        };
    }
}

Example 7 with StringComparator

use of org.apache.druid.query.ordering.StringComparator in project druid by druid-io.

the class DefaultLimitSpec method build.

@Override
public Function<Sequence<ResultRow>, Sequence<ResultRow>> build(final GroupByQuery query) {
    final List<DimensionSpec> dimensions = query.getDimensions();
    // Can avoid re-sorting if the natural ordering is good enough.
    boolean sortingNeeded = dimensions.size() < columns.size();
    final Set<String> aggAndPostAggNames = new HashSet<>();
    for (AggregatorFactory agg : query.getAggregatorSpecs()) {
        aggAndPostAggNames.add(agg.getName());
    }
    for (PostAggregator postAgg : query.getPostAggregatorSpecs()) {
        aggAndPostAggNames.add(postAgg.getName());
    }
    if (!sortingNeeded) {
        for (int i = 0; i < columns.size(); i++) {
            final OrderByColumnSpec columnSpec = columns.get(i);
            if (aggAndPostAggNames.contains(columnSpec.getDimension())) {
                sortingNeeded = true;
                break;
            }
            final ColumnType columnType = getOrderByType(columnSpec, dimensions);
            final StringComparator naturalComparator;
            if (columnType.is(ValueType.STRING)) {
                naturalComparator = StringComparators.LEXICOGRAPHIC;
            } else if (columnType.isNumeric()) {
                naturalComparator = StringComparators.NUMERIC;
            } else if (columnType.isArray()) {
                if (columnType.getElementType().isNumeric()) {
                    naturalComparator = StringComparators.NUMERIC;
                } else {
                    naturalComparator = StringComparators.LEXICOGRAPHIC;
                }
            } else {
                sortingNeeded = true;
                break;
            }
            if (columnSpec.getDirection() != OrderByColumnSpec.Direction.ASCENDING || !columnSpec.getDimensionComparator().equals(naturalComparator) || !columnSpec.getDimension().equals(dimensions.get(i).getOutputName())) {
                sortingNeeded = true;
                break;
            }
        }
    }
    if (!sortingNeeded) {
        // If granularity is ALL, sortByDimsFirst doesn't change the sorting order.
        sortingNeeded = !query.getGranularity().equals(Granularities.ALL) && query.getContextSortByDimsFirst();
    }
    if (!sortingNeeded) {
        String timestampField = query.getContextValue(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD);
        if (timestampField != null && !timestampField.isEmpty()) {
            int timestampResultFieldIndex = query.getContextValue(GroupByQuery.CTX_TIMESTAMP_RESULT_FIELD_INDEX);
            sortingNeeded = query.getContextSortByDimsFirst() ? timestampResultFieldIndex != query.getDimensions().size() - 1 : timestampResultFieldIndex != 0;
        }
    }
    final Function<Sequence<ResultRow>, Sequence<ResultRow>> sortAndLimitFn;
    if (sortingNeeded) {
        // Materialize the Comparator first for fast-fail error checking.
        final Ordering<ResultRow> ordering = makeComparator(query.getResultRowSignature(), query.getResultRowHasTimestamp(), query.getDimensions(), query.getAggregatorSpecs(), query.getPostAggregatorSpecs(), query.getContextSortByDimsFirst());
        // underlying data isn't changing. (Useful for query reproducibility and offset-based pagination.)
        if (isLimited()) {
            sortAndLimitFn = results -> new TopNSequence<>(results, ordering, limit + offset);
        } else {
            sortAndLimitFn = results -> Sequences.sort(results, ordering).limit(limit + offset);
        }
    } else {
        if (isLimited()) {
            sortAndLimitFn = results -> results.limit(limit + offset);
        } else {
            sortAndLimitFn = Functions.identity();
        }
    }
    // Finally, apply offset after sorting and limiting.
    if (isOffset()) {
        return results -> sortAndLimitFn.apply(results).skip(offset);
    } else {
        return sortAndLimitFn;
    }
}

Example 8 with StringComparator

use of org.apache.druid.query.ordering.StringComparator in project hive by apache.

the class DruidStorageHandlerUtils method toDruidFilter.

@Nullable
private static DimFilter toDruidFilter(ExprNodeDesc filterExpr, Configuration configuration, List<VirtualColumn> virtualColumns, boolean resolveDynamicValues) {
    if (filterExpr == null) {
        return null;
    }
    Class<? extends GenericUDF> genericUDFClass = getGenericUDFClassFromExprDesc(filterExpr);
    if (FunctionRegistry.isOpAnd(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new AndDimFilter(delegates);
        }
    }
    if (FunctionRegistry.isOpOr(filterExpr)) {
        Iterator<ExprNodeDesc> iterator = filterExpr.getChildren().iterator();
        List<DimFilter> delegates = Lists.newArrayList();
        while (iterator.hasNext()) {
            DimFilter filter = toDruidFilter(iterator.next(), configuration, virtualColumns, resolveDynamicValues);
            if (filter != null) {
                delegates.add(filter);
            }
        }
        if (!delegates.isEmpty()) {
            return new OrDimFilter(delegates);
        }
    } else if (GenericUDFBetween.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(1), virtualColumns);
        if (col != null) {
            try {
                StringComparator comparator = stringTypeInfos.contains(child.get(1).getTypeInfo()) ? StringComparators.LEXICOGRAPHIC : StringComparators.NUMERIC;
                String lower = evaluate(child.get(2), configuration, resolveDynamicValues);
                String upper = evaluate(child.get(3), configuration, resolveDynamicValues);
                return new BoundDimFilter(col, lower, upper, false, false, null, null, comparator);
            } catch (HiveException e) {
                throw new RuntimeException(e);
            }
        }
    } else if (GenericUDFInBloomFilter.class == genericUDFClass) {
        List<ExprNodeDesc> child = filterExpr.getChildren();
        String col = extractColName(child.get(0), virtualColumns);
        if (col != null) {
            try {
                BloomKFilter bloomFilter = evaluateBloomFilter(child.get(1), configuration, resolveDynamicValues);
                return new BloomDimFilter(col, BloomKFilterHolder.fromBloomKFilter(bloomFilter), null);
            } catch (HiveException | IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
    return null;
}

Example 9 with StringComparator

use of org.apache.druid.query.ordering.StringComparator in project druid by druid-io.

the class ComparableList method compareWithComparator.

public static int compareWithComparator(StringComparator stringComparator, ComparableList lhsComparableArray, ComparableList rhsComparableArray) {
    final StringComparator comparator = stringComparator == null ? StringComparators.NUMERIC : stringComparator;
    if (lhsComparableArray == null && rhsComparableArray == null) {
        return 0;
    } else if (lhsComparableArray == null) {
        return -1;
    } else if (rhsComparableArray == null) {
        return 1;
    }
    List lhs = lhsComparableArray.getDelegate();
    List rhs = rhsComparableArray.getDelegate();
    int minLength = Math.min(lhs.size(), rhs.size());
    // noinspection ArrayEquality
    if (lhs == rhs) {
        return 0;
    }
    for (int i = 0; i < minLength; i++) {
        final int cmp = comparator.compare(String.valueOf(lhs.get(i)), String.valueOf(rhs.get(i)));
        if (cmp == 0) {
            continue;
        }
        return cmp;
    }
    if (lhs.size() == rhs.size()) {
        return 0;
    } else if (lhs.size() < rhs.size()) {
        return -1;
    }
    return 1;
}

Example 10 with StringComparator

use of org.apache.druid.query.ordering.StringComparator in project druid by druid-io.

the class ComparableStringArray method compareWithComparator.

public static int compareWithComparator(StringComparator stringComparator, ComparableStringArray lhsComparableArray, ComparableStringArray rhsComparableArray) {
    final StringComparator comparator = stringComparator == null ? StringComparators.LEXICOGRAPHIC : stringComparator;
    if (lhsComparableArray == null && rhsComparableArray == null) {
        return 0;
    } else if (lhsComparableArray == null) {
        return -1;
    } else if (rhsComparableArray == null) {
        return 1;
    }
    String[] lhs = lhsComparableArray.getDelegate();
    String[] rhs = rhsComparableArray.getDelegate();
    int minLength = Math.min(lhs.length, rhs.length);
    // noinspection ArrayEquality
    if (lhs == rhs) {
        return 0;
    }
    for (int i = 0; i < minLength; i++) {
        final int cmp = comparator.compare(lhs[i], rhs[i]);
        if (cmp == 0) {
            continue;
        }
        return cmp;
    }
    if (lhs.length == rhs.length) {
        return 0;
    } else if (lhs.length < rhs.length) {
        return -1;
    }
    return 1;
}