Examples with CustomIndex org.locationtech.geowave.core.store.index.CustomIndex used on opensource projects

Example 1 with CustomIndex

use of org.locationtech.geowave.core.store.index.CustomIndex in project geowave by locationtech.

the class FilterConstraints method getIndexData.

/**
 * Get the multi-dimensional index data from these constraints.
 *
 * @return the multi-dimensional index data
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public List<MultiDimensionalIndexData<V>> getIndexData() {
    if (cachedIndexData == null) {
        if ((adapter == null) || (index == null) || (indexMapping == null)) {
            return Lists.newArrayList();
        }
        if (index instanceof CustomIndex) {
            final TextIndexStrategy indexStrategy = (TextIndexStrategy) ((CustomIndex) index).getCustomIndexStrategy();
            if (!(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter)) {
                throw new RuntimeException("Unable to determine adapter field used by text index.");
            }
            final String fieldName = ((AdapterFieldTextIndexEntryConverter) indexStrategy.getEntryConverter()).getFieldName();
            final IndexFieldConstraints<?> fieldConstraint = fieldConstraints.get(fieldName);
            final List<DimensionConstraints<String>> dimensionConstraints = Lists.newArrayList();
            if (fieldConstraint == null) {
                dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((String) null, (String) null, true, true, true))));
            } else if (fieldConstraint instanceof TextFieldConstraints) {
                final DimensionConstraints<String> dimensionConstraint = ((TextFieldConstraints) fieldConstraint).getDimensionRanges(0);
                if (dimensionConstraint == null) {
                    dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((String) null, (String) null, true, true, true))));
                } else {
                    dimensionConstraints.add(dimensionConstraint);
                }
            } else {
                throw new RuntimeException("Non-text field constraints cannot be used for a text index.");
            }
            cachedIndexData = (List) TextFieldConstraints.toIndexData(dimensionConstraints);
        } else {
            // Right now all index strategies that aren't custom are numeric
            final CommonIndexModel indexModel = index.getIndexModel();
            final int numStrategyDimensions = index.getIndexStrategy().getOrderedDimensionDefinitions().length;
            final List<DimensionConstraints<Double>> dimensionConstraints = Lists.newArrayListWithCapacity(numStrategyDimensions);
            final Map<String, Integer> indexFieldDimensions = Maps.newHashMap();
            final NumericDimensionField<?>[] dimensions = indexModel.getDimensions();
            int dimensionIndex = 0;
            for (final NumericDimensionField<?> indexField : dimensions) {
                if (dimensionIndex >= numStrategyDimensions) {
                    // Only build constraints for dimensions used by the index strategy.
                    break;
                }
                dimensionIndex++;
                final String indexFieldName = indexField.getFieldName();
                if (!indexFieldDimensions.containsKey(indexFieldName)) {
                    indexFieldDimensions.put(indexFieldName, 0);
                }
                final int indexFieldDimension = indexFieldDimensions.get(indexFieldName);
                final IndexFieldMapper<?, ?> mapper = indexMapping.getMapperForIndexField(indexFieldName);
                final String[] adapterFields = mapper.getIndexOrderedAdapterFields();
                IndexFieldConstraints<?> fieldConstraint = null;
                if (adapterFields.length > 1 && isSingleDimension(indexFieldName, dimensions)) {
                    // constraints
                    for (int i = 0; i < adapterFields.length; i++) {
                        final IndexFieldConstraints<?> constraint = fieldConstraints.get(adapterFields[i]);
                        if (fieldConstraint == null) {
                            fieldConstraint = constraint;
                        } else {
                            fieldConstraint.and((IndexFieldConstraints) constraint);
                        }
                    }
                } else {
                    fieldConstraint = fieldConstraints.get(adapterFields[indexFieldDimension % adapterFields.length]);
                }
                if (fieldConstraint == null) {
                    dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((Double) null, (Double) null, true, true, true))));
                } else if (fieldConstraint instanceof NumericFieldConstraints) {
                    final DimensionConstraints<Double> dimensionConstraint = ((NumericFieldConstraints) fieldConstraint).getDimensionRanges(indexFieldDimension % fieldConstraint.getDimensionCount());
                    if (dimensionConstraint == null) {
                        dimensionConstraints.add(DimensionConstraints.of(Lists.newArrayList(FilterRange.of((Double) null, (Double) null, true, true, true))));
                    } else {
                        dimensionConstraints.add(dimensionConstraint);
                    }
                    indexFieldDimensions.put(indexFieldName, indexFieldDimension + 1);
                } else {
                    throw new RuntimeException("Non-numeric field constraints cannot be used for a numeric index.");
                }
            }
            cachedIndexData = (List) NumericFieldConstraints.toIndexData(dimensionConstraints);
        }
    }
    return cachedIndexData;
}

Example 2 with CustomIndex

use of org.locationtech.geowave.core.store.index.CustomIndex in project geowave by locationtech.

the class OptimalExpressionQuery method determineBestIndices.

@SuppressWarnings({ "rawtypes", "unchecked" })
public List<Pair<Index, List<InternalDataAdapter<?>>>> determineBestIndices(final BaseQueryOptions baseOptions, final InternalDataAdapter<?>[] adapters, final AdapterIndexMappingStore adapterIndexMappingStore, final IndexStore indexStore, final DataStatisticsStore statisticsStore) {
    final Map<Index, List<InternalDataAdapter<?>>> bestIndices = Maps.newHashMap();
    final Set<String> referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    for (final InternalDataAdapter<?> adapter : adapters) {
        if (!adapterMatchesFilter(adapter, referencedFields)) {
            continue;
        }
        final AdapterToIndexMapping[] adapterIndices = adapterIndexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
        final Map<Index, FilterConstraints<?>> indexConstraints = Maps.newHashMap();
        Index bestIndex = null;
        for (final AdapterToIndexMapping mapping : adapterIndices) {
            if ((baseOptions.getIndexName() != null) && !baseOptions.getIndexName().equals(mapping.getIndexName())) {
                continue;
            }
            final Index index = mapping.getIndex(indexStore);
            if (indexFilter != null && !indexFilter.test(index)) {
                continue;
            }
            if ((bestIndex == null) || ((bestIndex instanceof AttributeIndex) && !(index instanceof AttributeIndex))) {
                bestIndex = index;
            }
            final Set<String> indexedFields = Sets.newHashSet();
            final Class<? extends Comparable> filterClass;
            if ((index instanceof CustomIndex) && (((CustomIndex<?, ?>) index).getCustomIndexStrategy() instanceof TextIndexStrategy)) {
                final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) index).getCustomIndexStrategy();
                if (!(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter)) {
                    continue;
                }
                indexedFields.add(((AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter()).getFieldName());
                filterClass = String.class;
            } else {
                for (final IndexFieldMapper<?, ?> mapper : mapping.getIndexFieldMappers()) {
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.add(adapterField);
                    }
                }
                // Remove any fields that are part of the common index model, but not used in the index
                // strategy. They shouldn't be considered when trying to find a best match. In the future
                // it may be useful to consider an index that has extra common index dimensions that
                // contain filtered fields over one that only matches indexed dimensions. For example, if
                // I have a spatial index, and a spatial index that stores time, it should pick the one
                // that stores time if I supply a temporal constraint, even though it isn't part of the
                // index strategy.
                final int modelDimensions = index.getIndexModel().getDimensions().length;
                final int strategyDimensions = index.getIndexStrategy().getOrderedDimensionDefinitions().length;
                for (int i = modelDimensions - 1; i >= strategyDimensions; i--) {
                    final IndexFieldMapper<?, ?> mapper = mapping.getMapperForIndexField(index.getIndexModel().getDimensions()[i].getFieldName());
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.remove(adapterField);
                    }
                }
                filterClass = Double.class;
            }
            if (referencedFields.containsAll(indexedFields)) {
                final FilterConstraints<?> constraints = filter.getConstraints(filterClass, statisticsStore, adapter, mapping, index, indexedFields);
                if (constraints.constrainsAllFields(indexedFields)) {
                    indexConstraints.put(index, constraints);
                }
            }
        }
        if (indexConstraints.size() == 1) {
            final Entry<Index, FilterConstraints<?>> bestEntry = indexConstraints.entrySet().iterator().next();
            bestIndex = bestEntry.getKey();
            constraintCache.put(adapter.getTypeName(), bestEntry.getValue());
        } else if (indexConstraints.size() > 1) {
            // determine which constraint is the best
            double bestCardinality = Double.MAX_VALUE;
            Index bestConstrainedIndex = null;
            for (final Entry<Index, FilterConstraints<?>> entry : indexConstraints.entrySet()) {
                final QueryRanges ranges = entry.getValue().getQueryRanges(baseOptions, statisticsStore);
                if (ranges.isEmpty()) {
                    continue;
                }
                // TODO: A future optimization would be to add a default numeric histogram for any numeric
                // index dimensions and just use the index data ranges to determine cardinality rather
                // than decomposing query ranges.
                final StatisticId<RowRangeHistogramValue> statisticId = IndexStatistic.generateStatisticId(entry.getKey().getName(), RowRangeHistogramStatistic.STATS_TYPE, Statistic.INTERNAL_TAG);
                final RowRangeHistogramStatistic histogram = (RowRangeHistogramStatistic) statisticsStore.getStatisticById(statisticId);
                final double cardinality = DataStoreUtils.cardinality(statisticsStore, histogram, adapter, bestConstrainedIndex, ranges);
                if ((bestConstrainedIndex == null) || (cardinality < bestCardinality)) {
                    bestConstrainedIndex = entry.getKey();
                    bestCardinality = cardinality;
                }
            }
            if (bestConstrainedIndex != null) {
                bestIndex = bestConstrainedIndex;
                constraintCache.put(adapter.getTypeName(), indexConstraints.get(bestIndex));
            }
        }
        if (bestIndex == null) {
            continue;
        }
        if (!bestIndices.containsKey(bestIndex)) {
            bestIndices.put(bestIndex, Lists.newArrayList());
        }
        bestIndices.get(bestIndex).add(adapter);
    }
    return bestIndices.entrySet().stream().map(e -> Pair.of(e.getKey(), e.getValue())).collect(Collectors.toList());
}

Example 3 with CustomIndex

use of org.locationtech.geowave.core.store.index.CustomIndex in project geowave by locationtech.

the class OptimalExpressionQuery method createQueryConstraints.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
public QueryConstraints createQueryConstraints(final InternalDataAdapter<?> adapter, final Index index, final AdapterToIndexMapping indexMapping) {
    if (!constraintCache.containsKey(adapter.getTypeName())) {
        filter.prepare(adapter, indexMapping, index);
        return new FilteredEverythingQuery(Lists.newArrayList(new ExpressionQueryFilter<>(filter, adapter, indexMapping)));
    }
    final Filter reduced = filter.removePredicatesForFields(constraintCache.get(adapter.getTypeName()).getExactConstrainedFields());
    final List<QueryFilter> filterList;
    if (reduced != null) {
        reduced.prepare(adapter, indexMapping, index);
        filterList = Lists.newArrayList(new ExpressionQueryFilter<>(reduced, adapter, indexMapping));
    } else {
        filterList = Lists.newArrayList();
    }
    if (index instanceof CustomIndex) {
        return new CustomQueryConstraints(new ExplicitTextSearch((List) constraintCache.get(adapter.getTypeName()).getIndexData()), filterList);
    }
    return new ExplicitFilteredQuery((List) constraintCache.get(adapter.getTypeName()).getIndexData(), filterList);
}

Example 4 with CustomIndex

use of org.locationtech.geowave.core.store.index.CustomIndex in project geowave by locationtech.

the class DataStoreUtils method constraintsToQueryRanges.

public static QueryRanges constraintsToQueryRanges(final List<MultiDimensionalNumericData> constraints, final Index index, final double[] targetResolutionPerDimensionForHierarchicalIndex, final int maxRanges, final IndexMetaData... hints) {
    if ((index instanceof CustomIndex) && (constraints != null) && (constraints.size() == 1) && (constraints.get(0) instanceof InternalCustomConstraints)) {
        return ((CustomIndex) index).getQueryRanges(((InternalCustomConstraints) constraints.get(0)).getCustomConstraints());
    }
    NumericIndexStrategy indexStrategy = index.getIndexStrategy();
    SubStrategy targetIndexStrategy = null;
    if ((targetResolutionPerDimensionForHierarchicalIndex != null) && (targetResolutionPerDimensionForHierarchicalIndex.length == indexStrategy.getOrderedDimensionDefinitions().length)) {
        // determine the correct tier to query for the given resolution
        final HierarchicalNumericIndexStrategy strategy = CompoundHierarchicalIndexStrategyWrapper.findHierarchicalStrategy(indexStrategy);
        if (strategy != null) {
            final TreeMap<Double, SubStrategy> sortedStrategies = new TreeMap<>();
            for (final SubStrategy subStrategy : strategy.getSubStrategies()) {
                final double[] idRangePerDimension = subStrategy.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                double rangeSum = 0;
                for (final double range : idRangePerDimension) {
                    rangeSum += range;
                }
                // sort by the sum of the range in each dimension
                sortedStrategies.put(rangeSum, subStrategy);
            }
            for (final SubStrategy subStrategy : sortedStrategies.descendingMap().values()) {
                final double[] highestPrecisionIdRangePerDimension = subStrategy.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                // if the id range is less than or equal to the target
                // resolution in each dimension, use this substrategy
                boolean withinTargetResolution = true;
                for (int d = 0; d < highestPrecisionIdRangePerDimension.length; d++) {
                    if (highestPrecisionIdRangePerDimension[d] > targetResolutionPerDimensionForHierarchicalIndex[d]) {
                        withinTargetResolution = false;
                        break;
                    }
                }
                if (withinTargetResolution) {
                    targetIndexStrategy = subStrategy;
                    break;
                }
            }
            if (targetIndexStrategy == null) {
                // if there is not a substrategy that is within the target
                // resolution, use the first substrategy (the lowest range
                // per dimension, which is the highest precision)
                targetIndexStrategy = sortedStrategies.firstEntry().getValue();
            }
            indexStrategy = targetIndexStrategy.getIndexStrategy();
        }
    }
    if ((constraints == null) || constraints.isEmpty()) {
        if (targetIndexStrategy != null) {
            // at least use the prefix of a substrategy if chosen
            return new QueryRanges(new byte[][] { targetIndexStrategy.getPrefix() });
        }
        // implies in negative and
        return new QueryRanges();
    // positive infinity
    } else {
        final List<QueryRanges> ranges = new ArrayList<>(constraints.size());
        for (final MultiDimensionalNumericData nd : constraints) {
            ranges.add(indexStrategy.getQueryRanges(nd, maxRanges, hints));
        }
        return ranges.size() > 1 ? new QueryRanges(ranges) : ranges.get(0);
    }
}

Example 5 with CustomIndex

use of org.locationtech.geowave.core.store.index.CustomIndex in project geowave by locationtech.

the class GeoWaveAttributeIndexIT method testTextAttributeIndex.

@Test
public void testTextAttributeIndex() {
    final DataStore ds = dataStore.createDataStore();
    final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
    final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    ds.addType(adapter, spatialIndex);
    Index textAttributeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, COMMENT_FIELD));
    ds.addIndex(TYPE_NAME, textAttributeIndex);
    textAttributeIndex = ds.getIndex(textAttributeIndex.getName());
    assertTrue(textAttributeIndex instanceof AttributeIndex);
    assertEquals(COMMENT_FIELD, ((AttributeIndex) textAttributeIndex).getAttributeName());
    assertTrue(textAttributeIndex instanceof CustomIndex);
    assertTrue(((CustomIndex<?, ?>) textAttributeIndex).getCustomIndexStrategy() instanceof TextIndexStrategy);
    final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) textAttributeIndex).getCustomIndexStrategy();
    assertTrue(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter);
    final AdapterFieldTextIndexEntryConverter<?> converter = (AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter();
    assertEquals(COMMENT_FIELD, converter.getFieldName());
    assertNotNull(converter.getAdapter());
    assertEquals(adapter.getTypeName(), converter.getAdapter().getTypeName());
    assertEquals(adapter.getFieldDescriptor(COMMENT_FIELD), converter.getAdapter().getFieldDescriptor(COMMENT_FIELD));
    final InternalAdapterStore adapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore mappingStore = dataStore.createAdapterIndexMappingStore();
    // Get the mapping for the attribute index
    final AdapterToIndexMapping mapping = mappingStore.getMapping(adapterStore.getAdapterId(adapter.getTypeName()), textAttributeIndex.getName());
    // The text index is a custom index, so there won't be any direct field mappings
    assertEquals(0, mapping.getIndexFieldMappers().size());
    // Ingest data
    ingestData(ds);
    // Query data from attribute index
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(textAttributeIndex.getName()).build())) {
        assertTrue(iterator.hasNext());
        // The null values are not indexed, so only 3/4 of the data should be present
        assertEquals((int) (TOTAL_FEATURES * 0.75), Iterators.size(iterator));
    }
    final Filter textFilter = TextFieldValue.of(COMMENT_FIELD).startsWith("c", true);
    // Query data from attribute index with a text constraint
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(textAttributeIndex.getName()).filter(textFilter).build())) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 4, Iterators.size(iterator));
    }
}