Search in sources :

Example 1 with FilterConstraints

use of org.locationtech.geowave.core.store.query.filter.expression.FilterConstraints in project geowave by locationtech.

the class OptimalExpressionQuery method determineBestIndices.

@SuppressWarnings({ "rawtypes", "unchecked" })
public List<Pair<Index, List<InternalDataAdapter<?>>>> determineBestIndices(final BaseQueryOptions baseOptions, final InternalDataAdapter<?>[] adapters, final AdapterIndexMappingStore adapterIndexMappingStore, final IndexStore indexStore, final DataStatisticsStore statisticsStore) {
    final Map<Index, List<InternalDataAdapter<?>>> bestIndices = Maps.newHashMap();
    final Set<String> referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    for (final InternalDataAdapter<?> adapter : adapters) {
        if (!adapterMatchesFilter(adapter, referencedFields)) {
            continue;
        }
        final AdapterToIndexMapping[] adapterIndices = adapterIndexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
        final Map<Index, FilterConstraints<?>> indexConstraints = Maps.newHashMap();
        Index bestIndex = null;
        for (final AdapterToIndexMapping mapping : adapterIndices) {
            if ((baseOptions.getIndexName() != null) && !baseOptions.getIndexName().equals(mapping.getIndexName())) {
                continue;
            }
            final Index index = mapping.getIndex(indexStore);
            if (indexFilter != null && !indexFilter.test(index)) {
                continue;
            }
            if ((bestIndex == null) || ((bestIndex instanceof AttributeIndex) && !(index instanceof AttributeIndex))) {
                bestIndex = index;
            }
            final Set<String> indexedFields = Sets.newHashSet();
            final Class<? extends Comparable> filterClass;
            if ((index instanceof CustomIndex) && (((CustomIndex<?, ?>) index).getCustomIndexStrategy() instanceof TextIndexStrategy)) {
                final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) index).getCustomIndexStrategy();
                if (!(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter)) {
                    continue;
                }
                indexedFields.add(((AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter()).getFieldName());
                filterClass = String.class;
            } else {
                for (final IndexFieldMapper<?, ?> mapper : mapping.getIndexFieldMappers()) {
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.add(adapterField);
                    }
                }
                // Remove any fields that are part of the common index model, but not used in the index
                // strategy. They shouldn't be considered when trying to find a best match. In the future
                // it may be useful to consider an index that has extra common index dimensions that
                // contain filtered fields over one that only matches indexed dimensions. For example, if
                // I have a spatial index, and a spatial index that stores time, it should pick the one
                // that stores time if I supply a temporal constraint, even though it isn't part of the
                // index strategy.
                final int modelDimensions = index.getIndexModel().getDimensions().length;
                final int strategyDimensions = index.getIndexStrategy().getOrderedDimensionDefinitions().length;
                for (int i = modelDimensions - 1; i >= strategyDimensions; i--) {
                    final IndexFieldMapper<?, ?> mapper = mapping.getMapperForIndexField(index.getIndexModel().getDimensions()[i].getFieldName());
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.remove(adapterField);
                    }
                }
                filterClass = Double.class;
            }
            if (referencedFields.containsAll(indexedFields)) {
                final FilterConstraints<?> constraints = filter.getConstraints(filterClass, statisticsStore, adapter, mapping, index, indexedFields);
                if (constraints.constrainsAllFields(indexedFields)) {
                    indexConstraints.put(index, constraints);
                }
            }
        }
        if (indexConstraints.size() == 1) {
            final Entry<Index, FilterConstraints<?>> bestEntry = indexConstraints.entrySet().iterator().next();
            bestIndex = bestEntry.getKey();
            constraintCache.put(adapter.getTypeName(), bestEntry.getValue());
        } else if (indexConstraints.size() > 1) {
            // determine which constraint is the best
            double bestCardinality = Double.MAX_VALUE;
            Index bestConstrainedIndex = null;
            for (final Entry<Index, FilterConstraints<?>> entry : indexConstraints.entrySet()) {
                final QueryRanges ranges = entry.getValue().getQueryRanges(baseOptions, statisticsStore);
                if (ranges.isEmpty()) {
                    continue;
                }
                // TODO: A future optimization would be to add a default numeric histogram for any numeric
                // index dimensions and just use the index data ranges to determine cardinality rather
                // than decomposing query ranges.
                final StatisticId<RowRangeHistogramValue> statisticId = IndexStatistic.generateStatisticId(entry.getKey().getName(), RowRangeHistogramStatistic.STATS_TYPE, Statistic.INTERNAL_TAG);
                final RowRangeHistogramStatistic histogram = (RowRangeHistogramStatistic) statisticsStore.getStatisticById(statisticId);
                final double cardinality = DataStoreUtils.cardinality(statisticsStore, histogram, adapter, bestConstrainedIndex, ranges);
                if ((bestConstrainedIndex == null) || (cardinality < bestCardinality)) {
                    bestConstrainedIndex = entry.getKey();
                    bestCardinality = cardinality;
                }
            }
            if (bestConstrainedIndex != null) {
                bestIndex = bestConstrainedIndex;
                constraintCache.put(adapter.getTypeName(), indexConstraints.get(bestIndex));
            }
        }
        if (bestIndex == null) {
            continue;
        }
        if (!bestIndices.containsKey(bestIndex)) {
            bestIndices.put(bestIndex, Lists.newArrayList());
        }
        bestIndices.get(bestIndex).add(adapter);
    }
    return bestIndices.entrySet().stream().map(e -> Pair.of(e.getKey(), e.getValue())).collect(Collectors.toList());
}
Also used : RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) IndexFilter(org.locationtech.geowave.core.store.index.IndexFilter) PersistenceUtils(org.locationtech.geowave.core.index.persist.PersistenceUtils) IndexFieldMapper(org.locationtech.geowave.core.store.api.IndexFieldMapper) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) LoggerFactory(org.slf4j.LoggerFactory) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) ByteBuffer(java.nio.ByteBuffer) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) Sets(com.beust.jcommander.internal.Sets) Lists(com.google.common.collect.Lists) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) Statistic(org.locationtech.geowave.core.store.api.Statistic) TextIndexStrategy(org.locationtech.geowave.core.index.text.TextIndexStrategy) VarintUtils(org.locationtech.geowave.core.index.VarintUtils) Index(org.locationtech.geowave.core.store.api.Index) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) BaseQueryOptions(org.locationtech.geowave.core.store.base.BaseQueryOptions) Logger(org.slf4j.Logger) AdapterFieldTextIndexEntryConverter(org.locationtech.geowave.core.store.index.TextAttributeIndexProvider.AdapterFieldTextIndexEntryConverter) Set(java.util.Set) IndexStatistic(org.locationtech.geowave.core.store.api.IndexStatistic) Collectors(java.util.stream.Collectors) Maps(com.google.common.collect.Maps) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) List(java.util.List) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) FilterConstraints(org.locationtech.geowave.core.store.query.filter.expression.FilterConstraints) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) Entry(java.util.Map.Entry) ExplicitTextSearch(org.locationtech.geowave.core.index.text.ExplicitTextSearch) RowRangeHistogramValue(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic.RowRangeHistogramValue) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) Index(org.locationtech.geowave.core.store.api.Index) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) Entry(java.util.Map.Entry) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) List(java.util.List) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterFieldTextIndexEntryConverter(org.locationtech.geowave.core.store.index.TextAttributeIndexProvider.AdapterFieldTextIndexEntryConverter) FilterConstraints(org.locationtech.geowave.core.store.query.filter.expression.FilterConstraints) TextIndexStrategy(org.locationtech.geowave.core.index.text.TextIndexStrategy) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex)

Aggregations

Sets (com.beust.jcommander.internal.Sets)1 Lists (com.google.common.collect.Lists)1 Maps (com.google.common.collect.Maps)1 ByteBuffer (java.nio.ByteBuffer)1 List (java.util.List)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Set (java.util.Set)1 Collectors (java.util.stream.Collectors)1 Pair (org.apache.commons.lang3.tuple.Pair)1 QueryRanges (org.locationtech.geowave.core.index.QueryRanges)1 VarintUtils (org.locationtech.geowave.core.index.VarintUtils)1 PersistenceUtils (org.locationtech.geowave.core.index.persist.PersistenceUtils)1 ExplicitTextSearch (org.locationtech.geowave.core.index.text.ExplicitTextSearch)1 TextIndexStrategy (org.locationtech.geowave.core.index.text.TextIndexStrategy)1 AdapterToIndexMapping (org.locationtech.geowave.core.store.AdapterToIndexMapping)1 AdapterIndexMappingStore (org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore)1 InternalDataAdapter (org.locationtech.geowave.core.store.adapter.InternalDataAdapter)1 AttributeIndex (org.locationtech.geowave.core.store.api.AttributeIndex)1 DataTypeAdapter (org.locationtech.geowave.core.store.api.DataTypeAdapter)1