use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.
the class GeoWaveAttributeIndexIT method testTemporalAttributeIndex.
@Test
public void testTemporalAttributeIndex() {
final DataStore ds = dataStore.createDataStore();
final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
ds.addType(adapter, spatialIndex);
Index temporalAttributeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, TIMESTAMP_FIELD));
ds.addIndex(TYPE_NAME, temporalAttributeIndex);
temporalAttributeIndex = ds.getIndex(temporalAttributeIndex.getName());
assertTrue(temporalAttributeIndex instanceof AttributeIndex);
assertEquals(TIMESTAMP_FIELD, ((AttributeIndex) temporalAttributeIndex).getAttributeName());
final InternalAdapterStore adapterStore = dataStore.createInternalAdapterStore();
final AdapterIndexMappingStore mappingStore = dataStore.createAdapterIndexMappingStore();
// Get the mapping for the attribute index
final AdapterToIndexMapping mapping = mappingStore.getMapping(adapterStore.getAdapterId(adapter.getTypeName()), temporalAttributeIndex.getName());
assertEquals(1, mapping.getIndexFieldMappers().size());
final IndexFieldMapper<?, ?> fieldMapper = mapping.getIndexFieldMappers().get(0);
assertEquals(Date.class, fieldMapper.adapterFieldType());
assertEquals(Long.class, fieldMapper.indexFieldType());
assertEquals(1, fieldMapper.getAdapterFields().length);
assertEquals(TIMESTAMP_FIELD, fieldMapper.getAdapterFields()[0]);
// Ingest data
ingestData(ds);
// Query data from attribute index
try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(temporalAttributeIndex.getName()).build())) {
assertTrue(iterator.hasNext());
// Half of the values are null and won't be indexed
assertEquals(TOTAL_FEATURES / 2, Iterators.size(iterator));
}
final Filter timeFilter = TemporalFieldValue.of(TIMESTAMP_FIELD).isBetween(new Date((long) (ONE_DAY_MILLIS * 10.5)), new Date((long) (ONE_DAY_MILLIS * 24.5)));
// Query data from attribute index with a numeric range constraint
try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(temporalAttributeIndex.getName()).filter(timeFilter).build())) {
assertTrue(iterator.hasNext());
assertEquals(7, Iterators.size(iterator));
}
}
use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.
the class OptimalExpressionQuery method determineBestIndices.
@SuppressWarnings({ "rawtypes", "unchecked" })
public List<Pair<Index, List<InternalDataAdapter<?>>>> determineBestIndices(final BaseQueryOptions baseOptions, final InternalDataAdapter<?>[] adapters, final AdapterIndexMappingStore adapterIndexMappingStore, final IndexStore indexStore, final DataStatisticsStore statisticsStore) {
final Map<Index, List<InternalDataAdapter<?>>> bestIndices = Maps.newHashMap();
final Set<String> referencedFields = Sets.newHashSet();
filter.addReferencedFields(referencedFields);
for (final InternalDataAdapter<?> adapter : adapters) {
if (!adapterMatchesFilter(adapter, referencedFields)) {
continue;
}
final AdapterToIndexMapping[] adapterIndices = adapterIndexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
final Map<Index, FilterConstraints<?>> indexConstraints = Maps.newHashMap();
Index bestIndex = null;
for (final AdapterToIndexMapping mapping : adapterIndices) {
if ((baseOptions.getIndexName() != null) && !baseOptions.getIndexName().equals(mapping.getIndexName())) {
continue;
}
final Index index = mapping.getIndex(indexStore);
if (indexFilter != null && !indexFilter.test(index)) {
continue;
}
if ((bestIndex == null) || ((bestIndex instanceof AttributeIndex) && !(index instanceof AttributeIndex))) {
bestIndex = index;
}
final Set<String> indexedFields = Sets.newHashSet();
final Class<? extends Comparable> filterClass;
if ((index instanceof CustomIndex) && (((CustomIndex<?, ?>) index).getCustomIndexStrategy() instanceof TextIndexStrategy)) {
final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) index).getCustomIndexStrategy();
if (!(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter)) {
continue;
}
indexedFields.add(((AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter()).getFieldName());
filterClass = String.class;
} else {
for (final IndexFieldMapper<?, ?> mapper : mapping.getIndexFieldMappers()) {
for (final String adapterField : mapper.getAdapterFields()) {
indexedFields.add(adapterField);
}
}
// Remove any fields that are part of the common index model, but not used in the index
// strategy. They shouldn't be considered when trying to find a best match. In the future
// it may be useful to consider an index that has extra common index dimensions that
// contain filtered fields over one that only matches indexed dimensions. For example, if
// I have a spatial index, and a spatial index that stores time, it should pick the one
// that stores time if I supply a temporal constraint, even though it isn't part of the
// index strategy.
final int modelDimensions = index.getIndexModel().getDimensions().length;
final int strategyDimensions = index.getIndexStrategy().getOrderedDimensionDefinitions().length;
for (int i = modelDimensions - 1; i >= strategyDimensions; i--) {
final IndexFieldMapper<?, ?> mapper = mapping.getMapperForIndexField(index.getIndexModel().getDimensions()[i].getFieldName());
for (final String adapterField : mapper.getAdapterFields()) {
indexedFields.remove(adapterField);
}
}
filterClass = Double.class;
}
if (referencedFields.containsAll(indexedFields)) {
final FilterConstraints<?> constraints = filter.getConstraints(filterClass, statisticsStore, adapter, mapping, index, indexedFields);
if (constraints.constrainsAllFields(indexedFields)) {
indexConstraints.put(index, constraints);
}
}
}
if (indexConstraints.size() == 1) {
final Entry<Index, FilterConstraints<?>> bestEntry = indexConstraints.entrySet().iterator().next();
bestIndex = bestEntry.getKey();
constraintCache.put(adapter.getTypeName(), bestEntry.getValue());
} else if (indexConstraints.size() > 1) {
// determine which constraint is the best
double bestCardinality = Double.MAX_VALUE;
Index bestConstrainedIndex = null;
for (final Entry<Index, FilterConstraints<?>> entry : indexConstraints.entrySet()) {
final QueryRanges ranges = entry.getValue().getQueryRanges(baseOptions, statisticsStore);
if (ranges.isEmpty()) {
continue;
}
// TODO: A future optimization would be to add a default numeric histogram for any numeric
// index dimensions and just use the index data ranges to determine cardinality rather
// than decomposing query ranges.
final StatisticId<RowRangeHistogramValue> statisticId = IndexStatistic.generateStatisticId(entry.getKey().getName(), RowRangeHistogramStatistic.STATS_TYPE, Statistic.INTERNAL_TAG);
final RowRangeHistogramStatistic histogram = (RowRangeHistogramStatistic) statisticsStore.getStatisticById(statisticId);
final double cardinality = DataStoreUtils.cardinality(statisticsStore, histogram, adapter, bestConstrainedIndex, ranges);
if ((bestConstrainedIndex == null) || (cardinality < bestCardinality)) {
bestConstrainedIndex = entry.getKey();
bestCardinality = cardinality;
}
}
if (bestConstrainedIndex != null) {
bestIndex = bestConstrainedIndex;
constraintCache.put(adapter.getTypeName(), indexConstraints.get(bestIndex));
}
}
if (bestIndex == null) {
continue;
}
if (!bestIndices.containsKey(bestIndex)) {
bestIndices.put(bestIndex, Lists.newArrayList());
}
bestIndices.get(bestIndex).add(adapter);
}
return bestIndices.entrySet().stream().map(e -> Pair.of(e.getKey(), e.getValue())).collect(Collectors.toList());
}
use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.
the class BaseDataStoreUtils method mapAdapterToIndex.
@SuppressWarnings({ "unchecked", "rawtypes" })
public static AdapterToIndexMapping mapAdapterToIndex(final InternalDataAdapter<?> adapter, final Index index) {
// Build up a list of index field mappers
final Map<String, IndexFieldMapper<?, ?>> mappers = Maps.newHashMap();
// Get index model dimensions
final NumericDimensionField<?>[] dimensions = index.getIndexModel().getDimensions();
// Map dimensions to index fields
final Map<String, List<NumericDimensionField>> indexFields = Arrays.stream(dimensions).collect(Collectors.groupingBy(dim -> dim.getFieldName(), Collectors.mapping(dim -> dim, Collectors.toList())));
// Get adapter fields
final FieldDescriptor<?>[] adapterFields = adapter.getFieldDescriptors();
for (final Entry<String, List<NumericDimensionField>> indexField : indexFields.entrySet()) {
// Get the hints used by all dimensions of the field
final Set<IndexDimensionHint> dimensionHints = Sets.newHashSet();
indexField.getValue().forEach(dim -> dimensionHints.addAll(dim.getDimensionHints()));
final Class<?> indexFieldClass = indexField.getValue().get(0).getFieldClass();
final String indexFieldName = indexField.getKey();
final IndexFieldOptions indexFieldOptions = indexField.getValue().get(0).getIndexFieldOptions();
// Get available mappers for the field class
final List<IndexFieldMapper<?, ?>> availableMappers = IndexFieldMapperRegistry.instance().getAvailableMappers(indexFieldClass);
if (availableMappers.size() == 0) {
throw new IllegalArgumentException("No index field mappers were found for the type: " + indexFieldClass.getName());
}
final List<FieldDescriptor<?>> hintedFields;
if (index instanceof AttributeIndex) {
// Only check the field that is set for the attribute index
hintedFields = Lists.newArrayList(adapter.getFieldDescriptor(((AttributeIndex) index).getAttributeName()));
} else {
// Get any adapter fields that have been hinted for this field
hintedFields = Arrays.stream(adapterFields).filter(field -> dimensionHints.stream().anyMatch(field.indexHints()::contains)).collect(Collectors.toList());
}
if (hintedFields.size() > 0) {
final Class<?> hintedFieldClass = hintedFields.get(0).bindingClass();
for (int i = 1; i < hintedFields.size(); i++) {
if (!hintedFieldClass.equals(hintedFields.get(i).bindingClass())) {
throw new IllegalArgumentException("All hinted fields must be of the same type.");
}
}
boolean mapperFound = false;
// Find a mapper that matches
for (final IndexFieldMapper<?, ?> mapper : availableMappers) {
if (mapper.isCompatibleWith(hintedFieldClass) && (mapper.adapterFieldCount() == hintedFields.size())) {
mapper.init(indexField.getKey(), (List) hintedFields, indexFieldOptions);
mappers.put(indexField.getKey(), mapper);
mapperFound = true;
break;
}
}
if (!mapperFound) {
throw new IllegalArgumentException("No registered index field mappers were found for the type: " + hintedFieldClass.getName() + "[" + hintedFields.size() + "] -> " + indexFieldClass.getName());
}
} else {
// Attempt to infer the field to use
// See if there are any suggested fields
boolean mapperFound = false;
for (final IndexFieldMapper<?, ?> mapper : availableMappers) {
final Set<String> suggestedFieldNames = mapper.getLowerCaseSuggestedFieldNames();
final List<FieldDescriptor<?>> matchingFields = Arrays.stream(adapterFields).filter(field -> mapper.isCompatibleWith(field.bindingClass()) && suggestedFieldNames.contains(field.fieldName().toLowerCase())).collect(Collectors.toList());
if (matchingFields.size() >= mapper.adapterFieldCount()) {
mapperFound = true;
mapper.init(indexFieldName, (List) matchingFields.stream().limit(mapper.adapterFieldCount()).collect(Collectors.toList()), indexFieldOptions);
mappers.put(indexFieldName, mapper);
break;
}
}
// See if a direct mapper is available
if (!mapperFound) {
for (final FieldDescriptor<?> fieldDescriptor : adapterFields) {
if (fieldDescriptor.bindingClass().equals(indexFieldClass)) {
final Optional<IndexFieldMapper<?, ?>> matchingMapper = availableMappers.stream().filter(mapper -> mapper.isCompatibleWith(fieldDescriptor.bindingClass()) && (mapper.adapterFieldCount() == 1)).findFirst();
if (matchingMapper.isPresent()) {
final IndexFieldMapper<?, ?> mapper = matchingMapper.get();
mapperFound = true;
mapper.init(indexFieldName, (List) Lists.newArrayList(fieldDescriptor), indexFieldOptions);
mappers.put(indexFieldName, mapper);
break;
}
}
}
}
// Check other mappers
if (!mapperFound) {
for (final IndexFieldMapper<?, ?> mapper : availableMappers) {
final List<FieldDescriptor<?>> matchingFields = Arrays.stream(adapterFields).filter(field -> mapper.isCompatibleWith(field.bindingClass())).collect(Collectors.toList());
if (matchingFields.size() >= mapper.adapterFieldCount()) {
mapperFound = true;
mapper.init(indexFieldName, (List) matchingFields.stream().limit(mapper.adapterFieldCount()).collect(Collectors.toList()), indexFieldOptions);
mappers.put(indexFieldName, mapper);
break;
}
}
}
if (!mapperFound) {
throw new IllegalArgumentException("No suitable index field mapper could be found for the index field " + indexFieldName);
}
}
}
return new AdapterToIndexMapping(adapter.getAdapterId(), index.getName(), mappers.values().stream().collect(Collectors.toList()));
}
use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.
the class GeoWaveAttributeIndexIT method testNumericAttributeIndex.
@Test
public void testNumericAttributeIndex() {
final DataStore ds = dataStore.createDataStore();
final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
ds.addType(adapter, spatialIndex);
Index integerAttributeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, INTEGER_FIELD));
ds.addIndex(TYPE_NAME, integerAttributeIndex);
integerAttributeIndex = ds.getIndex(integerAttributeIndex.getName());
assertTrue(integerAttributeIndex instanceof AttributeIndex);
assertEquals(INTEGER_FIELD, ((AttributeIndex) integerAttributeIndex).getAttributeName());
assertTrue(integerAttributeIndex.getIndexStrategy() instanceof SimpleIntegerIndexStrategy);
final InternalAdapterStore adapterStore = dataStore.createInternalAdapterStore();
final AdapterIndexMappingStore mappingStore = dataStore.createAdapterIndexMappingStore();
// Get the mapping for the attribute index
final AdapterToIndexMapping mapping = mappingStore.getMapping(adapterStore.getAdapterId(adapter.getTypeName()), integerAttributeIndex.getName());
assertEquals(1, mapping.getIndexFieldMappers().size());
final IndexFieldMapper<?, ?> fieldMapper = mapping.getIndexFieldMappers().get(0);
assertEquals(Integer.class, fieldMapper.adapterFieldType());
assertEquals(Integer.class, fieldMapper.indexFieldType());
assertEquals(1, fieldMapper.getAdapterFields().length);
assertEquals(INTEGER_FIELD, fieldMapper.getAdapterFields()[0]);
// Ingest data
ingestData(ds);
// Query data from attribute index
try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(integerAttributeIndex.getName()).build())) {
assertTrue(iterator.hasNext());
// Only one quarter of features should be indexed
assertEquals(TOTAL_FEATURES / 4, Iterators.size(iterator));
}
final Filter rangeFilter = NumericFieldValue.of(INTEGER_FIELD).isBetween(1.0, 40.0);
// Query data from attribute index with a numeric range constraint
try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(integerAttributeIndex.getName()).filter(rangeFilter).build())) {
assertTrue(iterator.hasNext());
assertEquals(10, Iterators.size(iterator));
}
}
use of org.locationtech.geowave.core.store.api.AttributeIndex in project geowave by locationtech.
the class GeoWaveAttributeIndexIT method testTextAttributeIndex.
@Test
public void testTextAttributeIndex() {
final DataStore ds = dataStore.createDataStore();
final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
ds.addType(adapter, spatialIndex);
Index textAttributeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, COMMENT_FIELD));
ds.addIndex(TYPE_NAME, textAttributeIndex);
textAttributeIndex = ds.getIndex(textAttributeIndex.getName());
assertTrue(textAttributeIndex instanceof AttributeIndex);
assertEquals(COMMENT_FIELD, ((AttributeIndex) textAttributeIndex).getAttributeName());
assertTrue(textAttributeIndex instanceof CustomIndex);
assertTrue(((CustomIndex<?, ?>) textAttributeIndex).getCustomIndexStrategy() instanceof TextIndexStrategy);
final TextIndexStrategy<?> indexStrategy = (TextIndexStrategy<?>) ((CustomIndex<?, ?>) textAttributeIndex).getCustomIndexStrategy();
assertTrue(indexStrategy.getEntryConverter() instanceof AdapterFieldTextIndexEntryConverter);
final AdapterFieldTextIndexEntryConverter<?> converter = (AdapterFieldTextIndexEntryConverter<?>) indexStrategy.getEntryConverter();
assertEquals(COMMENT_FIELD, converter.getFieldName());
assertNotNull(converter.getAdapter());
assertEquals(adapter.getTypeName(), converter.getAdapter().getTypeName());
assertEquals(adapter.getFieldDescriptor(COMMENT_FIELD), converter.getAdapter().getFieldDescriptor(COMMENT_FIELD));
final InternalAdapterStore adapterStore = dataStore.createInternalAdapterStore();
final AdapterIndexMappingStore mappingStore = dataStore.createAdapterIndexMappingStore();
// Get the mapping for the attribute index
final AdapterToIndexMapping mapping = mappingStore.getMapping(adapterStore.getAdapterId(adapter.getTypeName()), textAttributeIndex.getName());
// The text index is a custom index, so there won't be any direct field mappings
assertEquals(0, mapping.getIndexFieldMappers().size());
// Ingest data
ingestData(ds);
// Query data from attribute index
try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(textAttributeIndex.getName()).build())) {
assertTrue(iterator.hasNext());
// The null values are not indexed, so only 3/4 of the data should be present
assertEquals((int) (TOTAL_FEATURES * 0.75), Iterators.size(iterator));
}
final Filter textFilter = TextFieldValue.of(COMMENT_FIELD).startsWith("c", true);
// Query data from attribute index with a text constraint
try (CloseableIterator<SimpleFeature> iterator = ds.query(QueryBuilder.newBuilder(SimpleFeature.class).indexName(textAttributeIndex.getName()).filter(textFilter).build())) {
assertTrue(iterator.hasNext());
assertEquals(TOTAL_FEATURES / 4, Iterators.size(iterator));
}
}
Aggregations