Search in sources :

Example 1 with AttributeIndex

use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.

the class GeoWaveAttributeIndexIT method testTemporalAttributeIndex.

@Test
public void testTemporalAttributeIndex() {
    final DataStore ds = dataStore.createDataStore();
    final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
    final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    ds.addType(adapter, spatialIndex);
    Index temporalAttributeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, TIMESTAMP_FIELD));
    ds.addIndex(TYPE_NAME, temporalAttributeIndex);
    temporalAttributeIndex = ds.getIndex(temporalAttributeIndex.getName());
    assertTrue(temporalAttributeIndex instanceof AttributeIndex);
    assertEquals(TIMESTAMP_FIELD, ((AttributeIndex) temporalAttributeIndex).getAttributeName());
    final InternalAdapterStore adapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore mappingStore = dataStore.createAdapterIndexMappingStore();
    // Get the mapping for the attribute index
    final AdapterToIndexMapping mapping = mappingStore.getMapping(adapterStore.getAdapterId(adapter.getTypeName()), temporalAttributeIndex.getName());
    assertEquals(1, mapping.getIndexFieldMappers().size());
    final IndexFieldMapper<?, ?> fieldMapper = mapping.getIndexFieldMappers().get(0);
    assertEquals(Date.class, fieldMapper.adapterFieldType());
    assertEquals(Long.class, fieldMapper.indexFieldType());
    assertEquals(1, fieldMapper.getAdapterFields().length);
    assertEquals(TIMESTAMP_FIELD, fieldMapper.getAdapterFields()[0]);
    // Ingest data
    ingestData(ds);
    // Query data from attribute index
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(temporalAttributeIndex.getName()).build())) {
        assertTrue(iterator.hasNext());
        // Half of the values are null and won't be indexed
        assertEquals(TOTAL_FEATURES / 2, Iterators.size(iterator));
    }
    final Filter timeFilter = TemporalFieldValue.of(TIMESTAMP_FIELD).isBetween(new Date((long) (ONE_DAY_MILLIS * 10.5)), new Date((long) (ONE_DAY_MILLIS * 24.5)));
    // Query data from attribute index with a numeric range constraint
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(temporalAttributeIndex.getName()).filter(timeFilter).build())) {
        assertTrue(iterator.hasNext());
        assertEquals(7, Iterators.size(iterator));
    }
}
Also used : InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) Index(org.locationtech.geowave.core.store.api.Index) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) SpatialOptions(org.locationtech.geowave.core.geotime.index.SpatialOptions) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Date(java.util.Date) AttributeIndexOptions(org.locationtech.geowave.core.store.index.AttributeIndexOptions) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) DataStore(org.locationtech.geowave.core.store.api.DataStore) Test(org.junit.Test)

Example 2 with AttributeIndex

use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.

the class OptimalExpressionQuery method determineBestIndices.

@SuppressWarnings({ "rawtypes", "unchecked" })
public List<Pair<Index, List<InternalDataAdapter<?>>>> determineBestIndices(final BaseQueryOptions baseOptions, final InternalDataAdapter<?>[] adapters, final AdapterIndexMappingStore adapterIndexMappingStore, final IndexStore indexStore, final DataStatisticsStore statisticsStore) {
    final Map<Index, List<InternalDataAdapter<?>>> bestIndices = Maps.newHashMap();
    final Set<String> referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    for (final InternalDataAdapter<?> adapter : adapters) {
        if (!adapterMatchesFilter(adapter, referencedFields)) {
            continue;
        }
        final AdapterToIndexMapping[] adapterIndices = adapterIndexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
        final Map<Index, FilterConstraints<?>> indexConstraints = Maps.newHashMap();
        Index bestIndex = null;
        for (final AdapterToIndexMapping mapping : adapterIndices) {
            if ((baseOptions.getIndexName() != null) && !baseOptions.getIndexName().equals(mapping.getIndexName())) {
                continue;
            }
            final Index index = mapping.getIndex(indexStore);
            if (indexFilter != null && !indexFilter.test(index)) {
                continue;
            }
            if ((bestIndex == null) || ((bestIndex instanceof AttributeIndex) && !(index instanceof AttributeIndex))) {
                bestIndex = index;
            }
            final Set<String> indexedFields = Sets.newHashSet();
            final Class<? extends Comparable> filterClass;
            if ((index instanceof CustomIndex) && (((CustomIndex<?, ?>) index).getCustomIndexStrategy() instanceof TextIndexStrategy)) {
                final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) index).getCustomIndexStrategy();
                if (!(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter)) {
                    continue;
                }
                indexedFields.add(((AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter()).getFieldName());
                filterClass = String.class;
            } else {
                for (final IndexFieldMapper<?, ?> mapper : mapping.getIndexFieldMappers()) {
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.add(adapterField);
                    }
                }
                // Remove any fields that are part of the common index model, but not used in the index
                // strategy. They shouldn't be considered when trying to find a best match. In the future
                // it may be useful to consider an index that has extra common index dimensions that
                // contain filtered fields over one that only matches indexed dimensions. For example, if
                // I have a spatial index, and a spatial index that stores time, it should pick the one
                // that stores time if I supply a temporal constraint, even though it isn't part of the
                // index strategy.
                final int modelDimensions = index.getIndexModel().getDimensions().length;
                final int strategyDimensions = index.getIndexStrategy().getOrderedDimensionDefinitions().length;
                for (int i = modelDimensions - 1; i >= strategyDimensions; i--) {
                    final IndexFieldMapper<?, ?> mapper = mapping.getMapperForIndexField(index.getIndexModel().getDimensions()[i].getFieldName());
                    for (final String adapterField : mapper.getAdapterFields()) {
                        indexedFields.remove(adapterField);
                    }
                }
                filterClass = Double.class;
            }
            if (referencedFields.containsAll(indexedFields)) {
                final FilterConstraints<?> constraints = filter.getConstraints(filterClass, statisticsStore, adapter, mapping, index, indexedFields);
                if (constraints.constrainsAllFields(indexedFields)) {
                    indexConstraints.put(index, constraints);
                }
            }
        }
        if (indexConstraints.size() == 1) {
            final Entry<Index, FilterConstraints<?>> bestEntry = indexConstraints.entrySet().iterator().next();
            bestIndex = bestEntry.getKey();
            constraintCache.put(adapter.getTypeName(), bestEntry.getValue());
        } else if (indexConstraints.size() > 1) {
            // determine which constraint is the best
            double bestCardinality = Double.MAX_VALUE;
            Index bestConstrainedIndex = null;
            for (final Entry<Index, FilterConstraints<?>> entry : indexConstraints.entrySet()) {
                final QueryRanges ranges = entry.getValue().getQueryRanges(baseOptions, statisticsStore);
                if (ranges.isEmpty()) {
                    continue;
                }
                // TODO: A future optimization would be to add a default numeric histogram for any numeric
                // index dimensions and just use the index data ranges to determine cardinality rather
                // than decomposing query ranges.
                final StatisticId<RowRangeHistogramValue> statisticId = IndexStatistic.generateStatisticId(entry.getKey().getName(), RowRangeHistogramStatistic.STATS_TYPE, Statistic.INTERNAL_TAG);
                final RowRangeHistogramStatistic histogram = (RowRangeHistogramStatistic) statisticsStore.getStatisticById(statisticId);
                final double cardinality = DataStoreUtils.cardinality(statisticsStore, histogram, adapter, bestConstrainedIndex, ranges);
                if ((bestConstrainedIndex == null) || (cardinality < bestCardinality)) {
                    bestConstrainedIndex = entry.getKey();
                    bestCardinality = cardinality;
                }
            }
            if (bestConstrainedIndex != null) {
                bestIndex = bestConstrainedIndex;
                constraintCache.put(adapter.getTypeName(), indexConstraints.get(bestIndex));
            }
        }
        if (bestIndex == null) {
            continue;
        }
        if (!bestIndices.containsKey(bestIndex)) {
            bestIndices.put(bestIndex, Lists.newArrayList());
        }
        bestIndices.get(bestIndex).add(adapter);
    }
    return bestIndices.entrySet().stream().map(e -> Pair.of(e.getKey(), e.getValue())).collect(Collectors.toList());
}
Also used : RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) IndexFilter(org.locationtech.geowave.core.store.index.IndexFilter) PersistenceUtils(org.locationtech.geowave.core.index.persist.PersistenceUtils) IndexFieldMapper(org.locationtech.geowave.core.store.api.IndexFieldMapper) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) LoggerFactory(org.slf4j.LoggerFactory) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) ByteBuffer(java.nio.ByteBuffer) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) Sets(com.beust.jcommander.internal.Sets) Lists(com.google.common.collect.Lists) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) Statistic(org.locationtech.geowave.core.store.api.Statistic) TextIndexStrategy(org.locationtech.geowave.core.index.text.TextIndexStrategy) VarintUtils(org.locationtech.geowave.core.index.VarintUtils) Index(org.locationtech.geowave.core.store.api.Index) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) BaseQueryOptions(org.locationtech.geowave.core.store.base.BaseQueryOptions) Logger(org.slf4j.Logger) AdapterFieldTextIndexEntryConverter(org.locationtech.geowave.core.store.index.TextAttributeIndexProvider.AdapterFieldTextIndexEntryConverter) Set(java.util.Set) IndexStatistic(org.locationtech.geowave.core.store.api.IndexStatistic) Collectors(java.util.stream.Collectors) Maps(com.google.common.collect.Maps) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) List(java.util.List) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) FilterConstraints(org.locationtech.geowave.core.store.query.filter.expression.FilterConstraints) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) Entry(java.util.Map.Entry) ExplicitTextSearch(org.locationtech.geowave.core.index.text.ExplicitTextSearch) RowRangeHistogramValue(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic.RowRangeHistogramValue) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) Index(org.locationtech.geowave.core.store.api.Index) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) Entry(java.util.Map.Entry) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) List(java.util.List) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterFieldTextIndexEntryConverter(org.locationtech.geowave.core.store.index.TextAttributeIndexProvider.AdapterFieldTextIndexEntryConverter) FilterConstraints(org.locationtech.geowave.core.store.query.filter.expression.FilterConstraints) TextIndexStrategy(org.locationtech.geowave.core.index.text.TextIndexStrategy) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex)

Example 3 with AttributeIndex

use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.

the class BaseDataStoreUtils method mapAdapterToIndex.

@SuppressWarnings({ "unchecked", "rawtypes" })
public static AdapterToIndexMapping mapAdapterToIndex(final InternalDataAdapter<?> adapter, final Index index) {
    // Build up a list of index field mappers
    final Map<String, IndexFieldMapper<?, ?>> mappers = Maps.newHashMap();
    // Get index model dimensions
    final NumericDimensionField<?>[] dimensions = index.getIndexModel().getDimensions();
    // Map dimensions to index fields
    final Map<String, List<NumericDimensionField>> indexFields = Arrays.stream(dimensions).collect(Collectors.groupingBy(dim -> dim.getFieldName(), Collectors.mapping(dim -> dim, Collectors.toList())));
    // Get adapter fields
    final FieldDescriptor<?>[] adapterFields = adapter.getFieldDescriptors();
    for (final Entry<String, List<NumericDimensionField>> indexField : indexFields.entrySet()) {
        // Get the hints used by all dimensions of the field
        final Set<IndexDimensionHint> dimensionHints = Sets.newHashSet();
        indexField.getValue().forEach(dim -> dimensionHints.addAll(dim.getDimensionHints()));
        final Class<?> indexFieldClass = indexField.getValue().get(0).getFieldClass();
        final String indexFieldName = indexField.getKey();
        final IndexFieldOptions indexFieldOptions = indexField.getValue().get(0).getIndexFieldOptions();
        // Get available mappers for the field class
        final List<IndexFieldMapper<?, ?>> availableMappers = IndexFieldMapperRegistry.instance().getAvailableMappers(indexFieldClass);
        if (availableMappers.size() == 0) {
            throw new IllegalArgumentException("No index field mappers were found for the type: " + indexFieldClass.getName());
        }
        final List<FieldDescriptor<?>> hintedFields;
        if (index instanceof AttributeIndex) {
            // Only check the field that is set for the attribute index
            hintedFields = Lists.newArrayList(adapter.getFieldDescriptor(((AttributeIndex) index).getAttributeName()));
        } else {
            // Get any adapter fields that have been hinted for this field
            hintedFields = Arrays.stream(adapterFields).filter(field -> dimensionHints.stream().anyMatch(field.indexHints()::contains)).collect(Collectors.toList());
        }
        if (hintedFields.size() > 0) {
            final Class<?> hintedFieldClass = hintedFields.get(0).bindingClass();
            for (int i = 1; i < hintedFields.size(); i++) {
                if (!hintedFieldClass.equals(hintedFields.get(i).bindingClass())) {
                    throw new IllegalArgumentException("All hinted fields must be of the same type.");
                }
            }
            boolean mapperFound = false;
            // Find a mapper that matches
            for (final IndexFieldMapper<?, ?> mapper : availableMappers) {
                if (mapper.isCompatibleWith(hintedFieldClass) && (mapper.adapterFieldCount() == hintedFields.size())) {
                    mapper.init(indexField.getKey(), (List) hintedFields, indexFieldOptions);
                    mappers.put(indexField.getKey(), mapper);
                    mapperFound = true;
                    break;
                }
            }
            if (!mapperFound) {
                throw new IllegalArgumentException("No registered index field mappers were found for the type: " + hintedFieldClass.getName() + "[" + hintedFields.size() + "] -> " + indexFieldClass.getName());
            }
        } else {
            // Attempt to infer the field to use
            // See if there are any suggested fields
            boolean mapperFound = false;
            for (final IndexFieldMapper<?, ?> mapper : availableMappers) {
                final Set<String> suggestedFieldNames = mapper.getLowerCaseSuggestedFieldNames();
                final List<FieldDescriptor<?>> matchingFields = Arrays.stream(adapterFields).filter(field -> mapper.isCompatibleWith(field.bindingClass()) && suggestedFieldNames.contains(field.fieldName().toLowerCase())).collect(Collectors.toList());
                if (matchingFields.size() >= mapper.adapterFieldCount()) {
                    mapperFound = true;
                    mapper.init(indexFieldName, (List) matchingFields.stream().limit(mapper.adapterFieldCount()).collect(Collectors.toList()), indexFieldOptions);
                    mappers.put(indexFieldName, mapper);
                    break;
                }
            }
            // See if a direct mapper is available
            if (!mapperFound) {
                for (final FieldDescriptor<?> fieldDescriptor : adapterFields) {
                    if (fieldDescriptor.bindingClass().equals(indexFieldClass)) {
                        final Optional<IndexFieldMapper<?, ?>> matchingMapper = availableMappers.stream().filter(mapper -> mapper.isCompatibleWith(fieldDescriptor.bindingClass()) && (mapper.adapterFieldCount() == 1)).findFirst();
                        if (matchingMapper.isPresent()) {
                            final IndexFieldMapper<?, ?> mapper = matchingMapper.get();
                            mapperFound = true;
                            mapper.init(indexFieldName, (List) Lists.newArrayList(fieldDescriptor), indexFieldOptions);
                            mappers.put(indexFieldName, mapper);
                            break;
                        }
                    }
                }
            }
            // Check other mappers
            if (!mapperFound) {
                for (final IndexFieldMapper<?, ?> mapper : availableMappers) {
                    final List<FieldDescriptor<?>> matchingFields = Arrays.stream(adapterFields).filter(field -> mapper.isCompatibleWith(field.bindingClass())).collect(Collectors.toList());
                    if (matchingFields.size() >= mapper.adapterFieldCount()) {
                        mapperFound = true;
                        mapper.init(indexFieldName, (List) matchingFields.stream().limit(mapper.adapterFieldCount()).collect(Collectors.toList()), indexFieldOptions);
                        mappers.put(indexFieldName, mapper);
                        break;
                    }
                }
            }
            if (!mapperFound) {
                throw new IllegalArgumentException("No suitable index field mapper could be found for the index field " + indexFieldName);
            }
        }
    }
    return new AdapterToIndexMapping(adapter.getAdapterId(), index.getName(), mappers.values().stream().collect(Collectors.toList()));
}
Also used : Arrays(java.util.Arrays) ParameterException(com.beust.jcommander.ParameterException) SortedSet(java.util.SortedSet) DataWriter(org.locationtech.geowave.core.store.data.DataWriter) Maps(com.beust.jcommander.internal.Maps) BiFunction(java.util.function.BiFunction) CommonIndexAggregation(org.locationtech.geowave.core.store.query.aggregate.CommonIndexAggregation) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) LoggerFactory(org.slf4j.LoggerFactory) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) LazyReadPersistenceEncoding(org.locationtech.geowave.core.store.adapter.LazyReadPersistenceEncoding) Aggregation(org.locationtech.geowave.core.store.api.Aggregation) Collections2(com.google.common.collect.Collections2) ByteBuffer(java.nio.ByteBuffer) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) NumericDimensionField(org.locationtech.geowave.core.store.dimension.NumericDimensionField) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) PartialAsyncPersistenceEncoding(org.locationtech.geowave.core.store.adapter.PartialAsyncPersistenceEncoding) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) FieldInfo(org.locationtech.geowave.core.store.base.IntermediaryWriteEntryInfo.FieldInfo) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) Wrapper(org.locationtech.geowave.core.store.CloseableIterator.Wrapper) FieldDescriptor(org.locationtech.geowave.core.store.adapter.FieldDescriptor) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) VisibilityComposer(org.locationtech.geowave.core.store.data.visibility.VisibilityComposer) BitmaskedPairComparator(org.locationtech.geowave.core.store.flatten.BitmaskedPairComparator) Collection(java.util.Collection) Set(java.util.Set) IndexedAdapterPersistenceEncoding(org.locationtech.geowave.core.store.adapter.IndexedAdapterPersistenceEncoding) Collectors(java.util.stream.Collectors) IndexUtils(org.locationtech.geowave.core.index.IndexUtils) ScanCallback(org.locationtech.geowave.core.store.callback.ScanCallback) Sets(com.google.common.collect.Sets) List(java.util.List) VisibilityHandler(org.locationtech.geowave.core.store.api.VisibilityHandler) InsertionIds(org.locationtech.geowave.core.index.InsertionIds) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) BitmaskUtils(org.locationtech.geowave.core.store.flatten.BitmaskUtils) Entry(java.util.Map.Entry) Optional(java.util.Optional) FieldWriter(org.locationtech.geowave.core.store.data.field.FieldWriter) MultiDimensionalNumericData(org.locationtech.geowave.core.index.numeric.MultiDimensionalNumericData) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) IndexFieldMapperRegistry(org.locationtech.geowave.core.store.index.IndexFieldMapperRegistry) DataStoreProperty(org.locationtech.geowave.core.store.DataStoreProperty) StringUtils(org.locationtech.geowave.core.index.StringUtils) DataIndexUtils(org.locationtech.geowave.core.store.base.dataidx.DataIndexUtils) GeoWaveValueImpl(org.locationtech.geowave.core.store.entities.GeoWaveValueImpl) AdapterPersistenceEncoding(org.locationtech.geowave.core.store.adapter.AdapterPersistenceEncoding) IndexFieldMapper(org.locationtech.geowave.core.store.api.IndexFieldMapper) HashMap(java.util.HashMap) ArrayUtils(org.apache.commons.lang3.ArrayUtils) IndexDimensionHint(org.locationtech.geowave.core.index.IndexDimensionHint) DataIndexRetrieval(org.locationtech.geowave.core.store.base.dataidx.DataIndexRetrieval) Function(java.util.function.Function) TreeSet(java.util.TreeSet) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) FullAsyncPersistenceEncoding(org.locationtech.geowave.core.store.adapter.FullAsyncPersistenceEncoding) LinkedHashMap(java.util.LinkedHashMap) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) Lists(com.google.common.collect.Lists) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) Suppliers(com.google.common.base.Suppliers) VarintUtils(org.locationtech.geowave.core.index.VarintUtils) AsyncPersistenceEncoding(org.locationtech.geowave.core.store.adapter.AsyncPersistenceEncoding) IndexFieldOptions(org.locationtech.geowave.core.store.api.IndexFieldMapper.IndexFieldOptions) PropertyStore(org.locationtech.geowave.core.store.PropertyStore) LinkedList(java.util.LinkedList) BatchDataIndexRetrieval(org.locationtech.geowave.core.store.base.dataidx.BatchDataIndexRetrieval) Index(org.locationtech.geowave.core.store.api.Index) Nullable(javax.annotation.Nullable) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) Logger(org.slf4j.Logger) CommonIndexModel(org.locationtech.geowave.core.store.index.CommonIndexModel) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) IOException(java.io.IOException) GeoWaveValue(org.locationtech.geowave.core.store.entities.GeoWaveValue) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) AdapterException(org.locationtech.geowave.core.store.adapter.exceptions.AdapterException) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) CustomIndexStrategy(org.locationtech.geowave.core.index.CustomIndexStrategy) AdapterAndIndexBasedQueryConstraints(org.locationtech.geowave.core.store.query.constraints.AdapterAndIndexBasedQueryConstraints) Comparator(java.util.Comparator) Collections(java.util.Collections) NumericDimensionField(org.locationtech.geowave.core.store.dimension.NumericDimensionField) IndexFieldMapper(org.locationtech.geowave.core.store.api.IndexFieldMapper) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) FieldDescriptor(org.locationtech.geowave.core.store.adapter.FieldDescriptor) IndexDimensionHint(org.locationtech.geowave.core.index.IndexDimensionHint) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) IndexDimensionHint(org.locationtech.geowave.core.index.IndexDimensionHint) IndexFieldOptions(org.locationtech.geowave.core.store.api.IndexFieldMapper.IndexFieldOptions)

Example 4 with AttributeIndex

use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.

the class GeoWaveAttributeIndexIT method testNumericAttributeIndex.

@Test
public void testNumericAttributeIndex() {
    final DataStore ds = dataStore.createDataStore();
    final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
    final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    ds.addType(adapter, spatialIndex);
    Index integerAttributeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, INTEGER_FIELD));
    ds.addIndex(TYPE_NAME, integerAttributeIndex);
    integerAttributeIndex = ds.getIndex(integerAttributeIndex.getName());
    assertTrue(integerAttributeIndex instanceof AttributeIndex);
    assertEquals(INTEGER_FIELD, ((AttributeIndex) integerAttributeIndex).getAttributeName());
    assertTrue(integerAttributeIndex.getIndexStrategy() instanceof SimpleIntegerIndexStrategy);
    final InternalAdapterStore adapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore mappingStore = dataStore.createAdapterIndexMappingStore();
    // Get the mapping for the attribute index
    final AdapterToIndexMapping mapping = mappingStore.getMapping(adapterStore.getAdapterId(adapter.getTypeName()), integerAttributeIndex.getName());
    assertEquals(1, mapping.getIndexFieldMappers().size());
    final IndexFieldMapper<?, ?> fieldMapper = mapping.getIndexFieldMappers().get(0);
    assertEquals(Integer.class, fieldMapper.adapterFieldType());
    assertEquals(Integer.class, fieldMapper.indexFieldType());
    assertEquals(1, fieldMapper.getAdapterFields().length);
    assertEquals(INTEGER_FIELD, fieldMapper.getAdapterFields()[0]);
    // Ingest data
    ingestData(ds);
    // Query data from attribute index
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(integerAttributeIndex.getName()).build())) {
        assertTrue(iterator.hasNext());
        // Only one quarter of features should be indexed
        assertEquals(TOTAL_FEATURES / 4, Iterators.size(iterator));
    }
    final Filter rangeFilter = NumericFieldValue.of(INTEGER_FIELD).isBetween(1.0, 40.0);
    // Query data from attribute index with a numeric range constraint
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(integerAttributeIndex.getName()).filter(rangeFilter).build())) {
        assertTrue(iterator.hasNext());
        assertEquals(10, Iterators.size(iterator));
    }
}
Also used : InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) Index(org.locationtech.geowave.core.store.api.Index) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) SpatialOptions(org.locationtech.geowave.core.geotime.index.SpatialOptions) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) AttributeIndexOptions(org.locationtech.geowave.core.store.index.AttributeIndexOptions) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) DataStore(org.locationtech.geowave.core.store.api.DataStore) SimpleIntegerIndexStrategy(org.locationtech.geowave.core.index.simple.SimpleIntegerIndexStrategy) Test(org.junit.Test)

Example 5 with AttributeIndex

use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.

the class GeoWaveAttributeIndexIT method testTextAttributeIndex.

@Test
public void testTextAttributeIndex() {
    final DataStore ds = dataStore.createDataStore();
    final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
    final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    ds.addType(adapter, spatialIndex);
    Index textAttributeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, COMMENT_FIELD));
    ds.addIndex(TYPE_NAME, textAttributeIndex);
    textAttributeIndex = ds.getIndex(textAttributeIndex.getName());
    assertTrue(textAttributeIndex instanceof AttributeIndex);
    assertEquals(COMMENT_FIELD, ((AttributeIndex) textAttributeIndex).getAttributeName());
    assertTrue(textAttributeIndex instanceof CustomIndex);
    assertTrue(((CustomIndex<?, ?>) textAttributeIndex).getCustomIndexStrategy() instanceof TextIndexStrategy);
    final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) textAttributeIndex).getCustomIndexStrategy();
    assertTrue(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter);
    final AdapterFieldTextIndexEntryConverter<?> converter = (AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter();
    assertEquals(COMMENT_FIELD, converter.getFieldName());
    assertNotNull(converter.getAdapter());
    assertEquals(adapter.getTypeName(), converter.getAdapter().getTypeName());
    assertEquals(adapter.getFieldDescriptor(COMMENT_FIELD), converter.getAdapter().getFieldDescriptor(COMMENT_FIELD));
    final InternalAdapterStore adapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore mappingStore = dataStore.createAdapterIndexMappingStore();
    // Get the mapping for the attribute index
    final AdapterToIndexMapping mapping = mappingStore.getMapping(adapterStore.getAdapterId(adapter.getTypeName()), textAttributeIndex.getName());
    // The text index is a custom index, so there won't be any direct field mappings
    assertEquals(0, mapping.getIndexFieldMappers().size());
    // Ingest data
    ingestData(ds);
    // Query data from attribute index
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(textAttributeIndex.getName()).build())) {
        assertTrue(iterator.hasNext());
        // The null values are not indexed, so only 3/4 of the data should be present
        assertEquals((int) (TOTAL_FEATURES * 0.75), Iterators.size(iterator));
    }
    final Filter textFilter = TextFieldValue.of(COMMENT_FIELD).startsWith("c", true);
    // Query data from attribute index with a text constraint
    try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(textAttributeIndex.getName()).filter(textFilter).build())) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 4, Iterators.size(iterator));
    }
}
Also used : InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) AdapterFieldTextIndexEntryConverter(org.locationtech.geowave.core.store.index.TextAttributeIndexProvider.AdapterFieldTextIndexEntryConverter) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) Index(org.locationtech.geowave.core.store.api.Index) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) SpatialOptions(org.locationtech.geowave.core.geotime.index.SpatialOptions) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) AttributeIndexOptions(org.locationtech.geowave.core.store.index.AttributeIndexOptions) AttributeIndex(org.locationtech.geowave.core.store.api.AttributeIndex) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) DataStore(org.locationtech.geowave.core.store.api.DataStore) TextIndexStrategy(org.locationtech.geowave.core.index.text.TextIndexStrategy) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) Test(org.junit.Test)

Aggregations

AttributeIndex (org.locationtech.geowave.core.store.api.AttributeIndex)8 Index (org.locationtech.geowave.core.store.api.Index)8 AdapterToIndexMapping (org.locationtech.geowave.core.store.AdapterToIndexMapping)6 AdapterIndexMappingStore (org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore)6 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)5 CustomIndex (org.locationtech.geowave.core.store.index.CustomIndex)5 Filter (org.locationtech.geowave.core.store.query.filter.expression.Filter)5 Test (org.junit.Test)4 SpatialOptions (org.locationtech.geowave.core.geotime.index.SpatialOptions)4 DataStore (org.locationtech.geowave.core.store.api.DataStore)4 AttributeIndexOptions (org.locationtech.geowave.core.store.index.AttributeIndexOptions)4 SimpleFeature (org.opengis.feature.simple.SimpleFeature)3 Lists (com.google.common.collect.Lists)2 ByteBuffer (java.nio.ByteBuffer)2 List (java.util.List)2 Map (java.util.Map)2 Entry (java.util.Map.Entry)2 Set (java.util.Set)2 Collectors (java.util.stream.Collectors)2 AttributeIndexImpl (org.locationtech.geowave.core.store.index.AttributeIndexImpl)2