Examples with QueryRanges org.locationtech.geowave.core.index.QueryRanges used on opensource projects

Example 1 with QueryRanges

use of org.locationtech.geowave.core.index.QueryRanges in project geowave by locationtech.

the class ClusteringUtils method getGeoWaveRangesForQuery.

/*
   * Method takes in a polygon and generates the corresponding ranges in a GeoWave spatial index
   */
protected static QueryRanges getGeoWaveRangesForQuery(final Polygon polygon) {
    final Index index = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final QueryRanges ranges = DataStoreUtils.constraintsToQueryRanges(new ExplicitSpatialQuery(polygon).getIndexConstraints(index), index, null, -1);
    return ranges;
}

Example 2 with QueryRanges

use of org.locationtech.geowave.core.index.QueryRanges in project geowave by locationtech.

the class TextIndexUtils method getForwardQueryRanges.

public static QueryRanges getForwardQueryRanges(final String term, final boolean caseSensitive) {
    final byte[] forwardTermBytes = StringUtils.stringToBinary(caseSensitive ? term : term.toLowerCase());
    final List<SinglePartitionQueryRanges> retVal = new ArrayList<>(1);
    retVal.add(new SinglePartitionQueryRanges(caseSensitive ? FORWARD_INDEX_CASE_SENSITIVE_PARTITION_KEY : FORWARD_INDEX_CASE_INSENSITIVE_PARTITION_KEY, Collections.singletonList(new ByteArrayRange(forwardTermBytes, forwardTermBytes))));
    return new QueryRanges(retVal);
}

Example 3 with QueryRanges

use of org.locationtech.geowave.core.index.QueryRanges in project geowave by locationtech.

the class TextIndexUtils method getReverseQueryRanges.

public static QueryRanges getReverseQueryRanges(final String startTerm, final String endTerm, final boolean startInclusive, final boolean endInclusive, final boolean caseSensitive) {
    byte[] startBytes = StringUtils.stringToBinary(new StringBuilder(caseSensitive ? startTerm : endTerm.toLowerCase()).reverse().toString());
    if (!startInclusive) {
        startBytes = ByteArrayUtils.getNextPrefix(startBytes);
    }
    final byte[] endBytes = StringUtils.stringToBinary(new StringBuilder(caseSensitive ? endTerm : endTerm.toLowerCase()).reverse().toString());
    if (!endInclusive) {
        startBytes = ByteArrayUtils.getPreviousPrefix(startBytes);
    }
    final List<SinglePartitionQueryRanges> retVal = new ArrayList<>(1);
    retVal.add(new SinglePartitionQueryRanges(caseSensitive ? REVERSE_INDEX_CASE_SENSITIVE_PARTITION_KEY : REVERSE_INDEX_CASE_INSENSITIVE_PARTITION_KEY, Collections.singletonList(new ByteArrayRange(startBytes, endBytes))));
    return new QueryRanges(retVal);
}

Example 4 with QueryRanges

use of org.locationtech.geowave.core.index.QueryRanges in project geowave by locationtech.

the class ChooseBestMatchIndexQueryStrategy method getIndices.

@Override
public CloseableIterator<Index> getIndices(final DataStatisticsStore statisticsStore, final AdapterIndexMappingStore mappingStore, final QueryConstraints query, final Index[] indices, final InternalDataAdapter<?> adapter, final Map<QueryHint, Object> hints) {
    return new CloseableIterator<Index>() {

        Index nextIdx = null;

        boolean done = false;

        int i = 0;

        @Override
        public boolean hasNext() {
            long min = Long.MAX_VALUE;
            Index bestIdx = null;
            while (!done && (i < indices.length)) {
                nextIdx = indices[i++];
                if (nextIdx.getIndexStrategy().getOrderedDimensionDefinitions().length == 0) {
                    continue;
                }
                final List<MultiDimensionalNumericData> constraints = query.getIndexConstraints(nextIdx);
                RowRangeHistogramStatistic rowRangeHistogramStatistic = null;
                try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statisticsStore.getIndexStatistics(nextIdx, RowRangeHistogramStatistic.STATS_TYPE, Statistic.INTERNAL_TAG)) {
                    if (stats.hasNext()) {
                        final Statistic<?> statistic = stats.next();
                        if ((statistic instanceof RowRangeHistogramStatistic) && (statistic.getBinningStrategy() instanceof CompositeBinningStrategy) && ((CompositeBinningStrategy) statistic.getBinningStrategy()).isOfType(DataTypeBinningStrategy.class, PartitionBinningStrategy.class)) {
                            rowRangeHistogramStatistic = (RowRangeHistogramStatistic) statistic;
                        }
                    }
                }
                if (rowRangeHistogramStatistic == null) {
                    LOGGER.warn("Best Match Heuristic requires statistic RowRangeHistogramStatistics for each index to properly choose an index.");
                }
                if (IndexUtils.isFullTableScan(constraints)) {
                    // result in a full table scan
                    if (bestIdx == null) {
                        bestIdx = nextIdx;
                    }
                } else {
                    final int maxRangeDecomposition;
                    if (hints.containsKey(QueryHint.MAX_RANGE_DECOMPOSITION)) {
                        maxRangeDecomposition = (Integer) hints.get(QueryHint.MAX_RANGE_DECOMPOSITION);
                    } else {
                        LOGGER.warn("No max range decomposition hint was provided, this should be provided from the data store options");
                        maxRangeDecomposition = 2000;
                    }
                    final QueryRanges ranges = DataStoreUtils.constraintsToQueryRanges(constraints, nextIdx, null, maxRangeDecomposition);
                    final long temp = DataStoreUtils.cardinality(statisticsStore, rowRangeHistogramStatistic, adapter, nextIdx, ranges);
                    if (temp < min) {
                        bestIdx = nextIdx;
                        min = temp;
                    }
                }
            }
            nextIdx = bestIdx;
            done = true;
            return nextIdx != null;
        }

        @Override
        public Index next() throws NoSuchElementException {
            if (nextIdx == null) {
                throw new NoSuchElementException();
            }
            final Index returnVal = nextIdx;
            nextIdx = null;
            return returnVal;
        }

        @Override
        public void remove() {
        }

        @Override
        public void close() {
        }
    };
}

Example 5 with QueryRanges

use of org.locationtech.geowave.core.index.QueryRanges in project geowave by locationtech.

the class OptimalExpressionQuery method determineBestIndices.

@SuppressWarnings({ "rawtypes", "unchecked" })
public List<Pair<Index, List<InternalDataAdapter<?>>>> determineBestIndices(final BaseQueryOptions baseOptions, final InternalDataAdapter<?>[] adapters, final AdapterIndexMappingStore adapterIndexMappingStore, final IndexStore indexStore, final DataStatisticsStore statisticsStore) {
    final Map<Index, List<InternalDataAdapter<?>>> bestIndices = Maps.newHashMap();
    final Set<String> referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    for (final InternalDataAdapter<?> adapter : adapters) {
        if (!adapterMatchesFilter(adapter, referencedFields)) {
            continue;
        }
        final AdapterToIndexMapping[] adapterIndices = adapterIndexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
        final Map<Index, FilterConstraints<?>> indexConstraints = Maps.newHashMap();
        Index bestIndex = null;
        for (final AdapterToIndexMapping mapping : adapterIndices) {
            if ((baseOptions.getIndexName() != null) && !baseOptions.getIndexName().equals(mapping.getIndexName())) {
                continue;
            }
            final Index index = mapping.getIndex(indexStore);
            if (indexFilter != null && !indexFilter.test(index)) {
                continue;
            }
            if ((bestIndex == null) || ((bestIndex instanceof AttributeIndex) && !(index instanceof AttributeIndex))) {
                bestIndex = index;
            }
            final Set<String> indexedFields = Sets.newHashSet();
            final Class<? extends Comparable> filterClass;
            if ((index instanceof CustomIndex) && (((CustomIndex<?, ?>) index).getCustomIndexStrategy() instanceof TextIndexStrategy)) {
                final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) index).getCustomIndexStrategy();
                if (!(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter)) {
                    continue;
                }
                indexedFields.add(((AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter()).getFieldName());
                filterClass = String.class;
            } else {
                for (final IndexFieldMapper<?, ?> mapper : mapping.getIndexFieldMappers()) {
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.add(adapterField);
                    }
                }
                // Remove any fields that are part of the common index model, but not used in the index
                // strategy. They shouldn't be considered when trying to find a best match. In the future
                // it may be useful to consider an index that has extra common index dimensions that
                // contain filtered fields over one that only matches indexed dimensions. For example, if
                // I have a spatial index, and a spatial index that stores time, it should pick the one
                // that stores time if I supply a temporal constraint, even though it isn't part of the
                // index strategy.
                final int modelDimensions = index.getIndexModel().getDimensions().length;
                final int strategyDimensions = index.getIndexStrategy().getOrderedDimensionDefinitions().length;
                for (int i = modelDimensions - 1; i >= strategyDimensions; i--) {
                    final IndexFieldMapper<?, ?> mapper = mapping.getMapperForIndexField(index.getIndexModel().getDimensions()[i].getFieldName());
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.remove(adapterField);
                    }
                }
                filterClass = Double.class;
            }
            if (referencedFields.containsAll(indexedFields)) {
                final FilterConstraints<?> constraints = filter.getConstraints(filterClass, statisticsStore, adapter, mapping, index, indexedFields);
                if (constraints.constrainsAllFields(indexedFields)) {
                    indexConstraints.put(index, constraints);
                }
            }
        }
        if (indexConstraints.size() == 1) {
            final Entry<Index, FilterConstraints<?>> bestEntry = indexConstraints.entrySet().iterator().next();
            bestIndex = bestEntry.getKey();
            constraintCache.put(adapter.getTypeName(), bestEntry.getValue());
        } else if (indexConstraints.size() > 1) {
            // determine which constraint is the best
            double bestCardinality = Double.MAX_VALUE;
            Index bestConstrainedIndex = null;
            for (final Entry<Index, FilterConstraints<?>> entry : indexConstraints.entrySet()) {
                final QueryRanges ranges = entry.getValue().getQueryRanges(baseOptions, statisticsStore);
                if (ranges.isEmpty()) {
                    continue;
                }
                // TODO: A future optimization would be to add a default numeric histogram for any numeric
                // index dimensions and just use the index data ranges to determine cardinality rather
                // than decomposing query ranges.
                final StatisticId<RowRangeHistogramValue> statisticId = IndexStatistic.generateStatisticId(entry.getKey().getName(), RowRangeHistogramStatistic.STATS_TYPE, Statistic.INTERNAL_TAG);
                final RowRangeHistogramStatistic histogram = (RowRangeHistogramStatistic) statisticsStore.getStatisticById(statisticId);
                final double cardinality = DataStoreUtils.cardinality(statisticsStore, histogram, adapter, bestConstrainedIndex, ranges);
                if ((bestConstrainedIndex == null) || (cardinality < bestCardinality)) {
                    bestConstrainedIndex = entry.getKey();
                    bestCardinality = cardinality;
                }
            }
            if (bestConstrainedIndex != null) {
                bestIndex = bestConstrainedIndex;
                constraintCache.put(adapter.getTypeName(), indexConstraints.get(bestIndex));
            }
        }
        if (bestIndex == null) {
            continue;
        }
        if (!bestIndices.containsKey(bestIndex)) {
            bestIndices.put(bestIndex, Lists.newArrayList());
        }
        bestIndices.get(bestIndex).add(adapter);
    }
    return bestIndices.entrySet().stream().map(e -> Pair.of(e.getKey(), e.getValue())).collect(Collectors.toList());
}