Search in sources :

Example 1 with AdapterIndexMappingStore

use of org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore in project geowave by locationtech.

the class DataStoreUtils method mergeData.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
    final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
    try {
        final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> adapter : adapters) {
            if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
                mergingAdapters.put(adapter.getAdapterId(), adapter);
            }
        }
        final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
        final short[] adapterIds = new short[1];
        for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
            adapterIds[0] = adapter.getKey();
            paramsBuilder.adapterIds(adapterIds);
            try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
                final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
                final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
                while (iterator.hasNext()) {
                    iterator.next();
                }
            } catch (final Exception e) {
                LOGGER.error("Exception occurred while merging data.", e);
                throw new RuntimeException(e);
            }
        }
    } finally {
        try {
            deleter.close();
        } catch (final Exception e) {
            LOGGER.warn("Exception occurred when closing deleter.", e);
        }
    }
    return true;
}
Also used : GeoWaveKeyImpl(org.locationtech.geowave.core.store.entities.GeoWaveKeyImpl) Arrays(java.util.Arrays) ParameterException(com.beust.jcommander.ParameterException) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) HierarchicalNumericIndexStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy) FlattenedDataSet(org.locationtech.geowave.core.store.flatten.FlattenedDataSet) LoggerFactory(org.slf4j.LoggerFactory) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) PartitionBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.PartitionBinningStrategy) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) ByteBuffer(java.nio.ByteBuffer) FieldReader(org.locationtech.geowave.core.store.data.field.FieldReader) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) Lists(com.clearspring.analytics.util.Lists) NumericDimensionField(org.locationtech.geowave.core.store.dimension.NumericDimensionField) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) InternalCustomConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints.InternalCustomConstraints) Map(java.util.Map) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) GeoWaveKey(org.locationtech.geowave.core.store.entities.GeoWaveKey) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FlattenedUnreadDataSingleRow(org.locationtech.geowave.core.store.flatten.FlattenedUnreadDataSingleRow) FlattenedFieldInfo(org.locationtech.geowave.core.store.flatten.FlattenedFieldInfo) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Set(java.util.Set) UUID(java.util.UUID) DataTypeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy) List(java.util.List) VisibilityHandler(org.locationtech.geowave.core.store.api.VisibilityHandler) InsertionIds(org.locationtech.geowave.core.index.InsertionIds) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) BitmaskUtils(org.locationtech.geowave.core.store.flatten.BitmaskUtils) Entry(java.util.Map.Entry) MultiDimensionalNumericData(org.locationtech.geowave.core.index.numeric.MultiDimensionalNumericData) RowTransform(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter.RowTransform) UnconstrainedVisibilityHandler(org.locationtech.geowave.core.store.data.visibility.UnconstrainedVisibilityHandler) GeoWaveRowImpl(org.locationtech.geowave.core.store.entities.GeoWaveRowImpl) NumericIndexStrategy(org.locationtech.geowave.core.index.NumericIndexStrategy) RowRangeHistogramValue(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic.RowRangeHistogramValue) ByteArray(org.locationtech.geowave.core.index.ByteArray) IndexMetaData(org.locationtech.geowave.core.index.IndexMetaData) RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) StringUtils(org.locationtech.geowave.core.index.StringUtils) GeoWaveValueImpl(org.locationtech.geowave.core.store.entities.GeoWaveValueImpl) AdapterPersistenceEncoding(org.locationtech.geowave.core.store.adapter.AdapterPersistenceEncoding) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) SubStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy.SubStrategy) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) ArrayList(java.util.ArrayList) Mergeable(org.locationtech.geowave.core.index.Mergeable) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) VarintUtils(org.locationtech.geowave.core.index.VarintUtils) LinkedList(java.util.LinkedList) Index(org.locationtech.geowave.core.store.api.Index) MetadataDeleter(org.locationtech.geowave.core.store.operations.MetadataDeleter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) SinglePartitionInsertionIds(org.locationtech.geowave.core.index.SinglePartitionInsertionIds) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) Logger(org.slf4j.Logger) PersistentDataset(org.locationtech.geowave.core.store.data.PersistentDataset) CommonIndexModel(org.locationtech.geowave.core.store.index.CommonIndexModel) DataStore(org.locationtech.geowave.core.store.api.DataStore) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) GeoWaveValue(org.locationtech.geowave.core.store.entities.GeoWaveValue) Maps(com.google.common.collect.Maps) File(java.io.File) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) TreeMap(java.util.TreeMap) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) HintKey(org.locationtech.geowave.core.store.query.options.CommonQueryOptions.HintKey) ByteArrayUtils(org.locationtech.geowave.core.index.ByteArrayUtils) CustomIndexStrategy(org.locationtech.geowave.core.index.CustomIndexStrategy) FlattenedUnreadData(org.locationtech.geowave.core.store.flatten.FlattenedUnreadData) Collections(java.util.Collections) CompositeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) ParameterException(com.beust.jcommander.ParameterException) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter)

Example 2 with AdapterIndexMappingStore

use of org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore in project geowave by locationtech.

the class AccumuloUtils method getIterator.

private static CloseableIterator<Entry<Key, Value>> getIterator(final Connector connector, final String namespace, final Index index) throws AccumuloException, AccumuloSecurityException, IOException, TableNotFoundException {
    CloseableIterator<Entry<Key, Value>> iterator = null;
    final AccumuloOptions options = new AccumuloOptions();
    final AccumuloOperations operations = new AccumuloOperations(connector, namespace, new AccumuloOptions());
    final IndexStore indexStore = new IndexStoreImpl(operations, options);
    final PersistentAdapterStore adapterStore = new AdapterStoreImpl(operations, options);
    final AdapterIndexMappingStore mappingStore = new AdapterIndexMappingStoreImpl(operations, options);
    if (indexStore.indexExists(index.getName())) {
        final ScannerBase scanner = operations.createBatchScanner(index.getName());
        ((BatchScanner) scanner).setRanges(AccumuloUtils.byteArrayRangesToAccumuloRanges(null));
        final IteratorSetting iteratorSettings = new IteratorSetting(10, "GEOWAVE_WHOLE_ROW_ITERATOR", WholeRowIterator.class);
        scanner.addScanIterator(iteratorSettings);
        final Iterator<Entry<Key, Value>> it = new IteratorWrapper(adapterStore, mappingStore, index, scanner.iterator(), new QueryFilter[] { new DedupeFilter() });
        iterator = new CloseableIteratorWrapper<>(new ScannerClosableWrapper(scanner), it);
    }
    return iterator;
}
Also used : AccumuloOperations(org.locationtech.geowave.datastore.accumulo.operations.AccumuloOperations) ScannerBase(org.apache.accumulo.core.client.ScannerBase) BatchScanner(org.apache.accumulo.core.client.BatchScanner) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) IndexStoreImpl(org.locationtech.geowave.core.store.metadata.IndexStoreImpl) Entry(java.util.Map.Entry) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) AdapterIndexMappingStoreImpl(org.locationtech.geowave.core.store.metadata.AdapterIndexMappingStoreImpl) IteratorSetting(org.apache.accumulo.core.client.IteratorSetting) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) AccumuloOptions(org.locationtech.geowave.datastore.accumulo.config.AccumuloOptions) DedupeFilter(org.locationtech.geowave.core.store.query.filter.DedupeFilter) AdapterStoreImpl(org.locationtech.geowave.core.store.metadata.AdapterStoreImpl) IndexStore(org.locationtech.geowave.core.store.index.IndexStore)

Example 3 with AdapterIndexMappingStore

use of org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore in project geowave by locationtech.

the class ExpressionQueryIT method testTemporalExpressionQueriesTemporalIndex.

@Test
public void testTemporalExpressionQueriesTemporalIndex() {
    final DataStore ds = dataStore.createDataStore();
    final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
    final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final Index temporalIndex = TemporalDimensionalityTypeProvider.createIndexFromOptions(new TemporalOptions());
    ds.addType(adapter, spatialIndex, temporalIndex);
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore aimStore = dataStore.createAdapterIndexMappingStore();
    final IndexStore indexStore = dataStore.createIndexStore();
    final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
    final InternalDataAdapter<?> internalAdapter = adapterStore.getAdapter(internalAdapterStore.getAdapterId(TYPE_NAME));
    // Ingest data
    ingestData(ds);
    // ///////////////////////////////////////////////////
    // After
    // ///////////////////////////////////////////////////
    Query<SimpleFeature> query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isAfter(new Date(ONE_DAY_MILLIS * (TOTAL_FEATURES / 2)))).build();
    QueryConstraints queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    ExplicitFilteredQuery constraints = (ExplicitFilteredQuery) queryConstraints;
    List<QueryFilter> filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    Filter filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.After);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 2 - 1, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Before
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isBefore(ONE_DAY_MILLIS * (TOTAL_FEATURES / 2))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Before);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 2, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // During or After
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isDuringOrAfter(ONE_DAY_MILLIS * (TOTAL_FEATURES / 2))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof DuringOrAfter);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 2, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Before or During
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isBeforeOrDuring(ONE_DAY_MILLIS * (TOTAL_FEATURES / 2))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof BeforeOrDuring);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 2 + 1, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // During
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isDuring(Interval.of(Instant.ofEpochMilli(ONE_DAY_MILLIS * 5), Instant.ofEpochMilli(ONE_DAY_MILLIS * 10)))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof During);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(5, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Between
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isBetween(Instant.ofEpochMilli(ONE_DAY_MILLIS * 5), Instant.ofEpochMilli(ONE_DAY_MILLIS * 10))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof TemporalBetween);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(6, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Contains (inverse of During)
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TemporalLiteral.of(Interval.of(Instant.ofEpochMilli(ONE_DAY_MILLIS * 5), Instant.ofEpochMilli(ONE_DAY_MILLIS * 10))).contains(TIMESTAMP)).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof During);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(5, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Overlaps
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.overlaps(Interval.of(Instant.ofEpochMilli(ONE_DAY_MILLIS * 5), Instant.ofEpochMilli(ONE_DAY_MILLIS * 10)))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof TimeOverlaps);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(5, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Equal To
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isEqualTo(ONE_DAY_MILLIS * 12)).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof TemporalEqualTo);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(1, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Not Equal To
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isNotEqualTo(ONE_DAY_MILLIS * 12).and(TIMESTAMP.isNotEqualTo(ONE_DAY_MILLIS * 8))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof And);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES - 2, Iterators.size(iterator));
    }
}
Also used : Before(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.Before) Index(org.locationtech.geowave.core.store.api.Index) CustomQueryConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) BeforeOrDuring(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.BeforeOrDuring) During(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.During) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) TimeOverlaps(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.TimeOverlaps) DuringOrAfter(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.DuringOrAfter) DataStore(org.locationtech.geowave.core.store.api.DataStore) TemporalEqualTo(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.TemporalEqualTo) ExplicitFilteredQuery(org.locationtech.geowave.core.store.query.constraints.ExplicitFilteredQuery) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) SpatialOptions(org.locationtech.geowave.core.geotime.index.SpatialOptions) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Date(java.util.Date) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) And(org.locationtech.geowave.core.store.query.filter.expression.And) SpatialTemporalOptions(org.locationtech.geowave.core.geotime.index.SpatialTemporalOptions) TemporalOptions(org.locationtech.geowave.core.geotime.index.TemporalOptions) BeforeOrDuring(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.BeforeOrDuring) TemporalBetween(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.TemporalBetween) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) Test(org.junit.Test)

Example 4 with AdapterIndexMappingStore

use of org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore in project geowave by locationtech.

the class ExpressionQueryIT method testIndexSelection.

@Test
public void testIndexSelection() {
    final DataStore ds = dataStore.createDataStore();
    final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
    final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final Index spatialTemporalIndex = SpatialTemporalDimensionalityTypeProvider.createIndexFromOptions(new SpatialTemporalOptions());
    final Index temporalIndex = TemporalDimensionalityTypeProvider.createIndexFromOptions(new TemporalOptions());
    ds.addType(adapter, spatialIndex, spatialTemporalIndex, temporalIndex);
    final Index altIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, ALTERNATE_GEOMETRY_FIELD));
    final Index integerIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, INTEGER_FIELD));
    final Index commentIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, COMMENT_FIELD));
    final Index idIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, ID_FIELD));
    final Index latitudeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, LATITUDE_FIELD));
    final Index longitudeIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, LONGITUDE_FIELD));
    ds.addIndex(TYPE_NAME, altIndex, integerIndex, commentIndex, idIndex, latitudeIndex, longitudeIndex);
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore aimStore = dataStore.createAdapterIndexMappingStore();
    final IndexStore indexStore = dataStore.createIndexStore();
    final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
    final InternalDataAdapter<?> internalAdapter = adapterStore.getAdapter(internalAdapterStore.getAdapterId(TYPE_NAME));
    // Ingest data
    ingestData(ds);
    // ///////////////////////////////////////////////////
    // Basic BBOX on Alternate Geometry
    // ///////////////////////////////////////////////////
    Query<SimpleFeature> query = QueryBuilder.newBuilder(SimpleFeature.class).filter(ALT.bbox(-64.5, 0.5, 0.5, 64.5)).build();
    QueryConstraints queryConstraints = assertBestIndex(internalAdapter, altIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    ExplicitFilteredQuery constraints = (ExplicitFilteredQuery) queryConstraints;
    List<QueryFilter> filters = constraints.createFilters(altIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    Filter filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof BBox);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 4, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Both bbox and comment are indexed, but comment
    // should result in fewer rows queried so that
    // should be the selected index
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(ALT.bbox(-64.5, -32.5, 32.5, 64.5).and(COMMENT.startsWith("b", true))).build();
    queryConstraints = assertBestIndex(internalAdapter, commentIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof CustomQueryConstraints);
    CustomQueryConstraints<?> customConstraints = (CustomQueryConstraints<?>) queryConstraints;
    filters = customConstraints.createFilters(commentIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // Comment predicate was exact so only the bbox filter should need to be performed
    assertTrue(filter instanceof BBox);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        int count = 0;
        while (iterator.hasNext()) {
            final SimpleFeature feature = iterator.next();
            assertEquals("Bravo", feature.getAttribute(COMMENT_FIELD));
            count++;
        }
        // 1/4 of entries match the comment predicate, but only 3/4 of those match the bounding box
        assertEquals(Math.round(TOTAL_FEATURES / 8 * 1.5), count);
    }
    // ///////////////////////////////////////////////////
    // Both bbox and comment are indexed, but bbox should
    // result in fewer rows queried so that should be the
    // selected index
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(ALT.bbox(-64.5, 32.5, -32.5, 64.5).and(COMMENT.startsWith("b", true))).build();
    queryConstraints = assertBestIndex(internalAdapter, altIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(altIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // bbox is not exact, so it will still be part of the filter
    assertTrue(filter instanceof And);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 16, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Latitude is the most constraining
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(LATITUDE.isGreaterThan(5).and(LATITUDE.isLessThan(10), LONGITUDE.isGreaterThanOrEqualTo(7))).build();
    queryConstraints = assertBestIndex(internalAdapter, latitudeIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(latitudeIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // since the latitude fields are exact, only the longitude needs to be filtered later
    assertTrue(filter instanceof ComparisonOperator);
    Set<String> referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(1, referencedFields.size());
    assertTrue(referencedFields.contains(LONGITUDE_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(3, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Longitude is the most constraining
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(LONGITUDE.isGreaterThanOrEqualTo(5).and(LONGITUDE.isLessThan(10).or(LATITUDE.isLessThanOrEqualTo(15)))).build();
    queryConstraints = assertBestIndex(internalAdapter, longitudeIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(longitudeIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // The second half of the expression cannot be predetermined, so both sides of the Or need to be
    // present
    assertTrue(filter instanceof Or);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(2, referencedFields.size());
    assertTrue(referencedFields.contains(LATITUDE_FIELD));
    assertTrue(referencedFields.contains(LONGITUDE_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(11, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Longitude is an exact range
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(LONGITUDE.isLessThan(-31.5).or(LONGITUDE.isGreaterThan(31.5))).build();
    queryConstraints = assertBestIndex(internalAdapter, longitudeIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(longitudeIndex);
    // The constraints are exact, so there shouldn't be any additional filtering
    assertEquals(0, filters.size());
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 2 + 1, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Default geom only should select spatial index
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.bbox(0.5, 0.5, 10.5, 10.5)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // BBox isn't exact, so it should still be filtered
    assertTrue(filter instanceof BBox);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(1, referencedFields.size());
    assertTrue(referencedFields.contains(DEFAULT_GEOMETRY_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(10, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Default geom and time should select spatial-
    // temporal index
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.bbox(0.5, 0.5, 30.5, 30.5).and(TIMESTAMP.isBefore(new Date((long) (66 * ONE_DAY_MILLIS + 1))))).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialTemporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialTemporalIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // BBox isn't exact, and neither is timestamp in a binned temporal index
    assertTrue(filter instanceof And);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(2, referencedFields.size());
    assertTrue(referencedFields.contains(DEFAULT_GEOMETRY_FIELD));
    assertTrue(referencedFields.contains(TIMESTAMP_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(2, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Only timestamp should use temporal index
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(TIMESTAMP.isBefore(new Date((long) (66 * ONE_DAY_MILLIS + 1)))).build();
    queryConstraints = assertBestIndex(internalAdapter, temporalIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(temporalIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // Timestamp is not exact on temporal index because there could be ranges
    assertTrue(filter instanceof Before);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(1, referencedFields.size());
    assertTrue(referencedFields.contains(TIMESTAMP_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(67, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Integer is more constraining, half of the ID
    // values end with 0
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(ID.endsWith("0").and(INTEGER.isBetween(10, 20))).build();
    queryConstraints = assertBestIndex(internalAdapter, integerIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(integerIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // Integer is exact, so only the string predicate should remain
    assertTrue(filter instanceof EndsWith);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(1, referencedFields.size());
    assertTrue(referencedFields.contains(ID_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(6, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // ID is more constraining
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(ID.endsWith("a0").and(INTEGER.isBetween(0, 40))).build();
    queryConstraints = assertBestIndex(internalAdapter, idIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof CustomQueryConstraints);
    customConstraints = (CustomQueryConstraints<?>) queryConstraints;
    filters = customConstraints.createFilters(commentIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // ID constraint is exact, so only the integer predicate should remain
    assertTrue(filter instanceof Between);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(1, referencedFields.size());
    assertTrue(referencedFields.contains(INTEGER_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(2, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Alternate geometry is 50% null, so it is more
    // constraining
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.bbox(-30.5, -30.5, 30.5, 30.5).and(ALT.bbox(-30.5, -30.5, 30.5, 30.5))).build();
    queryConstraints = assertBestIndex(internalAdapter, altIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(altIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // Neither bbox is exact, so they will both be filtered
    assertTrue(filter instanceof And);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(2, referencedFields.size());
    assertTrue(referencedFields.contains(DEFAULT_GEOMETRY_FIELD));
    assertTrue(referencedFields.contains(ALTERNATE_GEOMETRY_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(30, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Integer is more constraining
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(INTEGER.isLessThan(-60).and(LATITUDE.isLessThan(5))).build();
    queryConstraints = assertBestIndex(internalAdapter, integerIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(integerIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    // Neither bbox is exact, so they will both be filtered
    assertTrue(filter instanceof ComparisonOperator);
    referencedFields = Sets.newHashSet();
    filter.addReferencedFields(referencedFields);
    assertEquals(1, referencedFields.size());
    assertTrue(referencedFields.contains(LATITUDE_FIELD));
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(4, Iterators.size(iterator));
    }
}
Also used : Before(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.Before) ComparisonOperator(org.locationtech.geowave.core.store.query.filter.expression.ComparisonOperator) Or(org.locationtech.geowave.core.store.query.filter.expression.Or) Index(org.locationtech.geowave.core.store.api.Index) CustomQueryConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) LineString(org.locationtech.jts.geom.LineString) SpatialTemporalOptions(org.locationtech.geowave.core.geotime.index.SpatialTemporalOptions) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) DataStore(org.locationtech.geowave.core.store.api.DataStore) ExplicitFilteredQuery(org.locationtech.geowave.core.store.query.constraints.ExplicitFilteredQuery) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) TemporalBetween(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.TemporalBetween) Between(org.locationtech.geowave.core.store.query.filter.expression.Between) SpatialOptions(org.locationtech.geowave.core.geotime.index.SpatialOptions) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Point(org.locationtech.jts.geom.Point) Disjoint(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.Disjoint) Date(java.util.Date) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) AttributeIndexOptions(org.locationtech.geowave.core.store.index.AttributeIndexOptions) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) BBox(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.BBox) And(org.locationtech.geowave.core.store.query.filter.expression.And) SpatialTemporalOptions(org.locationtech.geowave.core.geotime.index.SpatialTemporalOptions) TemporalOptions(org.locationtech.geowave.core.geotime.index.TemporalOptions) CustomQueryConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) EndsWith(org.locationtech.geowave.core.store.query.filter.expression.text.EndsWith) Test(org.junit.Test)

Example 5 with AdapterIndexMappingStore

use of org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore in project geowave by locationtech.

the class ExpressionQueryIT method testSpatialExpressionQueries.

@Test
public void testSpatialExpressionQueries() {
    final DataStore ds = dataStore.createDataStore();
    final DataTypeAdapter<SimpleFeature> adapter = createDataAdapter();
    final Index spatialIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    ds.addType(adapter, spatialIndex);
    final Index altIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, ALTERNATE_GEOMETRY_FIELD));
    final Index polyIndex = AttributeDimensionalityTypeProvider.createIndexFromOptions(ds, new AttributeIndexOptions(TYPE_NAME, POLYGON_FIELD));
    ds.addIndex(TYPE_NAME, altIndex, polyIndex);
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore aimStore = dataStore.createAdapterIndexMappingStore();
    final IndexStore indexStore = dataStore.createIndexStore();
    final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
    final InternalDataAdapter<?> internalAdapter = adapterStore.getAdapter(internalAdapterStore.getAdapterId(TYPE_NAME));
    final Polygon boxPoly = GeometryUtils.GEOMETRY_FACTORY.createPolygon(new Coordinate[] { new Coordinate(-20.5, -20.5), new Coordinate(-20.5, 20.5), new Coordinate(20.5, 20.5), new Coordinate(20.5, -20.5), new Coordinate(-20.5, -20.5) });
    final Polygon boxPoly2 = GeometryUtils.GEOMETRY_FACTORY.createPolygon(new Coordinate[] { new Coordinate(-20, -20), new Coordinate(-20, 20), new Coordinate(20, 20), new Coordinate(20, -20), new Coordinate(-20, -20) });
    // Large diagonal line
    final LineString line = GeometryUtils.GEOMETRY_FACTORY.createLineString(new Coordinate[] { new Coordinate(-20.5, -20.5), new Coordinate(20.5, 20.5) });
    // Ingest data
    ingestData(ds);
    // ///////////////////////////////////////////////////
    // Basic BBOX
    // ///////////////////////////////////////////////////
    Query<SimpleFeature> query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.bbox(0.5, 0.5, 64.5, 64.5)).build();
    QueryConstraints queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    ExplicitFilteredQuery constraints = (ExplicitFilteredQuery) queryConstraints;
    List<QueryFilter> filters = constraints.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    Filter filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof BBox);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 2 - 1, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Loose BBOX
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.bboxLoose(0.5, 0.5, 64.5, 64.5)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialIndex);
    assertEquals(0, filters.size());
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES / 2 - 1, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Intersects
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(ALT.intersects(boxPoly)).build();
    queryConstraints = assertBestIndex(internalAdapter, altIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(altIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Intersects);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(20, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Loose Intersects
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(ALT.intersectsLoose(boxPoly)).build();
    queryConstraints = assertBestIndex(internalAdapter, altIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(altIndex);
    assertEquals(0, filters.size());
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(20, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Disjoint
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.disjoint(boxPoly)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof FilteredEverythingQuery);
    FilteredEverythingQuery everything = (FilteredEverythingQuery) queryConstraints;
    filters = everything.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Disjoint);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES - 41, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Loose Disjoint
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.disjointLoose(boxPoly)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof FilteredEverythingQuery);
    everything = (FilteredEverythingQuery) queryConstraints;
    filters = everything.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Disjoint);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES - 41, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Crosses
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(POLY.crosses(line)).build();
    queryConstraints = assertBestIndex(internalAdapter, polyIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(polyIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Crosses);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(43, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Overlaps
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(POLY.overlaps(boxPoly)).build();
    queryConstraints = assertBestIndex(internalAdapter, polyIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(polyIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Overlaps);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        // it overlaps 2 polygons in each corner
        assertEquals(4, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Contains
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.contains(boxPoly)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof SpatialContains);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertFalse(iterator.hasNext());
    }
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(SpatialLiteral.of(boxPoly).contains(GEOM)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof SpatialContains);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(41, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Touches
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(POLY.touches(boxPoly2)).build();
    queryConstraints = assertBestIndex(internalAdapter, polyIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(polyIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Touches);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(2, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // Within
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.within(boxPoly)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Within);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(41, Iterators.size(iterator));
    }
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(SpatialLiteral.of(boxPoly).within(GEOM)).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof Within);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertFalse(iterator.hasNext());
    }
    // ///////////////////////////////////////////////////
    // EqualTo
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.isEqualTo(GeometryUtils.GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)))).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof ExplicitFilteredQuery);
    constraints = (ExplicitFilteredQuery) queryConstraints;
    filters = constraints.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof SpatialEqualTo);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(1, Iterators.size(iterator));
    }
    // ///////////////////////////////////////////////////
    // NotEqualTo
    // ///////////////////////////////////////////////////
    query = QueryBuilder.newBuilder(SimpleFeature.class).filter(GEOM.isNotEqualTo(GeometryUtils.GEOMETRY_FACTORY.createPoint(new Coordinate(1, 1)))).build();
    queryConstraints = assertBestIndex(internalAdapter, spatialIndex, query, adapterStore, internalAdapterStore, aimStore, indexStore, statsStore);
    assertTrue(queryConstraints instanceof FilteredEverythingQuery);
    everything = (FilteredEverythingQuery) queryConstraints;
    filters = everything.createFilters(spatialIndex);
    assertEquals(1, filters.size());
    assertTrue(filters.get(0) instanceof ExpressionQueryFilter);
    filter = ((ExpressionQueryFilter<?>) filters.get(0)).getFilter();
    assertTrue(filter instanceof SpatialNotEqualTo);
    // Query data
    try (CloseableIterator<SimpleFeature> iterator = ds.query(query)) {
        assertTrue(iterator.hasNext());
        assertEquals(TOTAL_FEATURES - 1, Iterators.size(iterator));
    }
}
Also used : TimeOverlaps(org.locationtech.geowave.core.geotime.store.query.filter.expression.temporal.TimeOverlaps) Overlaps(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.Overlaps) Index(org.locationtech.geowave.core.store.api.Index) CustomQueryConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) Crosses(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.Crosses) DataStore(org.locationtech.geowave.core.store.api.DataStore) Intersects(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.Intersects) Disjoint(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.Disjoint) Touches(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.Touches) FilteredEverythingQuery(org.locationtech.geowave.core.store.query.constraints.FilteredEverythingQuery) Polygon(org.locationtech.jts.geom.Polygon) ExplicitFilteredQuery(org.locationtech.geowave.core.store.query.constraints.ExplicitFilteredQuery) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) SpatialNotEqualTo(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.SpatialNotEqualTo) SpatialOptions(org.locationtech.geowave.core.geotime.index.SpatialOptions) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) AttributeIndexOptions(org.locationtech.geowave.core.store.index.AttributeIndexOptions) SpatialEqualTo(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.SpatialEqualTo) SpatialContains(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.SpatialContains) Coordinate(org.locationtech.jts.geom.Coordinate) LineString(org.locationtech.jts.geom.LineString) QueryFilter(org.locationtech.geowave.core.store.query.filter.QueryFilter) ExpressionQueryFilter(org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter) Filter(org.locationtech.geowave.core.store.query.filter.expression.Filter) BBox(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.BBox) Within(org.locationtech.geowave.core.geotime.store.query.filter.expression.spatial.Within) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) Test(org.junit.Test)

Aggregations

AdapterIndexMappingStore (org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore)24 Index (org.locationtech.geowave.core.store.api.Index)21 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)19 PersistentAdapterStore (org.locationtech.geowave.core.store.adapter.PersistentAdapterStore)17 IndexStore (org.locationtech.geowave.core.store.index.IndexStore)15 DataStore (org.locationtech.geowave.core.store.api.DataStore)14 Test (org.junit.Test)12 DataStatisticsStore (org.locationtech.geowave.core.store.statistics.DataStatisticsStore)12 SimpleFeature (org.opengis.feature.simple.SimpleFeature)12 SpatialOptions (org.locationtech.geowave.core.geotime.index.SpatialOptions)11 AdapterToIndexMapping (org.locationtech.geowave.core.store.AdapterToIndexMapping)10 AttributeIndexOptions (org.locationtech.geowave.core.store.index.AttributeIndexOptions)10 QueryConstraints (org.locationtech.geowave.core.store.query.constraints.QueryConstraints)10 QueryFilter (org.locationtech.geowave.core.store.query.filter.QueryFilter)9 Filter (org.locationtech.geowave.core.store.query.filter.expression.Filter)9 InternalDataAdapter (org.locationtech.geowave.core.store.adapter.InternalDataAdapter)8 ExpressionQueryFilter (org.locationtech.geowave.core.store.query.filter.ExpressionQueryFilter)8 CustomQueryConstraints (org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints)7 IOException (java.io.IOException)6 AttributeIndex (org.locationtech.geowave.core.store.api.AttributeIndex)6