Search in sources :

Example 1 with InternalDataAdapter

use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.

the class DataStoreUtils method mergeData.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
    final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
    try {
        final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> adapter : adapters) {
            if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
                mergingAdapters.put(adapter.getAdapterId(), adapter);
            }
        }
        final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
        final short[] adapterIds = new short[1];
        for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
            adapterIds[0] = adapter.getKey();
            paramsBuilder.adapterIds(adapterIds);
            try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
                final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
                final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
                while (iterator.hasNext()) {
                    iterator.next();
                }
            } catch (final Exception e) {
                LOGGER.error("Exception occurred while merging data.", e);
                throw new RuntimeException(e);
            }
        }
    } finally {
        try {
            deleter.close();
        } catch (final Exception e) {
            LOGGER.warn("Exception occurred when closing deleter.", e);
        }
    }
    return true;
}
Also used : GeoWaveKeyImpl(org.locationtech.geowave.core.store.entities.GeoWaveKeyImpl) Arrays(java.util.Arrays) ParameterException(com.beust.jcommander.ParameterException) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) HierarchicalNumericIndexStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy) FlattenedDataSet(org.locationtech.geowave.core.store.flatten.FlattenedDataSet) LoggerFactory(org.slf4j.LoggerFactory) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) PartitionBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.PartitionBinningStrategy) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) ByteBuffer(java.nio.ByteBuffer) FieldReader(org.locationtech.geowave.core.store.data.field.FieldReader) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) Lists(com.clearspring.analytics.util.Lists) NumericDimensionField(org.locationtech.geowave.core.store.dimension.NumericDimensionField) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) InternalCustomConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints.InternalCustomConstraints) Map(java.util.Map) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) GeoWaveKey(org.locationtech.geowave.core.store.entities.GeoWaveKey) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FlattenedUnreadDataSingleRow(org.locationtech.geowave.core.store.flatten.FlattenedUnreadDataSingleRow) FlattenedFieldInfo(org.locationtech.geowave.core.store.flatten.FlattenedFieldInfo) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Set(java.util.Set) UUID(java.util.UUID) DataTypeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy) List(java.util.List) VisibilityHandler(org.locationtech.geowave.core.store.api.VisibilityHandler) InsertionIds(org.locationtech.geowave.core.index.InsertionIds) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) BitmaskUtils(org.locationtech.geowave.core.store.flatten.BitmaskUtils) Entry(java.util.Map.Entry) MultiDimensionalNumericData(org.locationtech.geowave.core.index.numeric.MultiDimensionalNumericData) RowTransform(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter.RowTransform) UnconstrainedVisibilityHandler(org.locationtech.geowave.core.store.data.visibility.UnconstrainedVisibilityHandler) GeoWaveRowImpl(org.locationtech.geowave.core.store.entities.GeoWaveRowImpl) NumericIndexStrategy(org.locationtech.geowave.core.index.NumericIndexStrategy) RowRangeHistogramValue(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic.RowRangeHistogramValue) ByteArray(org.locationtech.geowave.core.index.ByteArray) IndexMetaData(org.locationtech.geowave.core.index.IndexMetaData) RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) StringUtils(org.locationtech.geowave.core.index.StringUtils) GeoWaveValueImpl(org.locationtech.geowave.core.store.entities.GeoWaveValueImpl) AdapterPersistenceEncoding(org.locationtech.geowave.core.store.adapter.AdapterPersistenceEncoding) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) SubStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy.SubStrategy) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) ArrayList(java.util.ArrayList) Mergeable(org.locationtech.geowave.core.index.Mergeable) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) VarintUtils(org.locationtech.geowave.core.index.VarintUtils) LinkedList(java.util.LinkedList) Index(org.locationtech.geowave.core.store.api.Index) MetadataDeleter(org.locationtech.geowave.core.store.operations.MetadataDeleter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) SinglePartitionInsertionIds(org.locationtech.geowave.core.index.SinglePartitionInsertionIds) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) Logger(org.slf4j.Logger) PersistentDataset(org.locationtech.geowave.core.store.data.PersistentDataset) CommonIndexModel(org.locationtech.geowave.core.store.index.CommonIndexModel) DataStore(org.locationtech.geowave.core.store.api.DataStore) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) GeoWaveValue(org.locationtech.geowave.core.store.entities.GeoWaveValue) Maps(com.google.common.collect.Maps) File(java.io.File) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) TreeMap(java.util.TreeMap) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) HintKey(org.locationtech.geowave.core.store.query.options.CommonQueryOptions.HintKey) ByteArrayUtils(org.locationtech.geowave.core.index.ByteArrayUtils) CustomIndexStrategy(org.locationtech.geowave.core.index.CustomIndexStrategy) FlattenedUnreadData(org.locationtech.geowave.core.store.flatten.FlattenedUnreadData) Collections(java.util.Collections) CompositeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) ParameterException(com.beust.jcommander.ParameterException) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter)

Example 2 with InternalDataAdapter

use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.

the class GeoWaveSparkIngestIT method testBasicSparkIngest.

@Test
public void testBasicSparkIngest() throws Exception {
    // ingest test points
    TestUtils.testSparkIngest(dataStore, DimensionalityType.SPATIAL, S3URL, GDELT_INPUT_FILES, "gdelt");
    final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        // query by the full bounding box, make sure there is more than
        // 0 count and make sure the count matches the number of results
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", new Integer(GDELT_COUNT), new Integer(resultCount));
    }
    // Clean up
    TestUtils.deleteAll(dataStore);
}
Also used : GeometryFactory(org.locationtech.jts.geom.GeometryFactory) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) Envelope(org.locationtech.jts.geom.Envelope) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) Geometry(org.locationtech.jts.geom.Geometry) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) ExplicitSpatialQuery(org.locationtech.geowave.core.geotime.store.query.ExplicitSpatialQuery) CountValue(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Test(org.junit.Test)

Example 3 with InternalDataAdapter

use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.

the class BasicKafkaIT method testBasicIngestGpx.

@Test
public void testBasicIngestGpx() throws Exception {
    KafkaTestUtils.testKafkaStage(OSM_GPX_INPUT_DIR);
    KafkaTestUtils.testKafkaIngest(dataStorePluginOptions, false, OSM_GPX_INPUT_DIR);
    final DataStatisticsStore statsStore = dataStorePluginOptions.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStorePluginOptions.createAdapterStore();
    int adapterCount = 0;
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", EXPECTED_COUNT_PER_ADAPTER_ID.get(adapter.getTypeName()), new Integer(resultCount));
        adapterCount++;
    }
    assertTrue("There should be exactly two adapters", (adapterCount == 2));
}
Also used : GeometryFactory(org.locationtech.jts.geom.GeometryFactory) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) Envelope(org.locationtech.jts.geom.Envelope) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) Geometry(org.locationtech.jts.geom.Geometry) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) ExplicitSpatialQuery(org.locationtech.geowave.core.geotime.store.query.ExplicitSpatialQuery) CountValue(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Test(org.junit.Test)

Example 4 with InternalDataAdapter

use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testDeleteCQL.

protected void testDeleteCQL(final String cqlStr, final Index index) throws Exception {
    LOGGER.info("bulk deleting using CQL: '" + cqlStr + "'");
    final DataStore geowaveStore = getDataStorePluginOptions().createDataStore();
    // Retrieve the feature adapter for the CQL query generator
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> adapter : adapters) {
        // Create the CQL query
        final QueryConstraints query = OptimalCQLQuery.createOptimalQuery(cqlStr, (InternalGeotoolsFeatureDataAdapter<SimpleFeature>) adapter, null, null);
        deleteInternal(geowaveStore, index, query);
    }
}
Also used : PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) DataStore(org.locationtech.geowave.core.store.api.DataStore) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) SimpleFeature(org.opengis.feature.simple.SimpleFeature)

Example 5 with InternalDataAdapter

use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testQuery.

protected void testQuery(final URL savedFilterResource, final URL[] expectedResultsResources, final Pair<String, String> optimalCqlQueryGeometryAndTimeFields, final Index index, final String queryDescription, final CoordinateReferenceSystem crs, final boolean countDuplicates) throws IOException {
    LOGGER.info("querying " + queryDescription);
    final DataStore geowaveStore = getDataStorePluginOptions().createDataStore();
    // this file is the filtered dataset (using the previous file as a
    // filter) so use it to ensure the query worked
    final QueryConstraints constraints = TestUtils.resourceToQuery(savedFilterResource, optimalCqlQueryGeometryAndTimeFields, true);
    QueryBuilder<?, ?> bldr = QueryBuilder.newBuilder();
    if (index != null) {
        bldr = bldr.indexName(index.getName());
    }
    try (final CloseableIterator<?> actualResults = geowaveStore.query(bldr.constraints(constraints).build())) {
        final ExpectedResults expectedResults = TestUtils.getExpectedResults(expectedResultsResources, crs);
        int totalResults = 0;
        final List<Long> actualCentroids = new ArrayList<>();
        while (actualResults.hasNext()) {
            final Object obj = actualResults.next();
            if (obj instanceof SimpleFeature) {
                final SimpleFeature result = (SimpleFeature) obj;
                final long actualHashCentroid = TestUtils.hashCentroid((Geometry) result.getDefaultGeometry());
                Assert.assertTrue("Actual result '" + result.toString() + "' not found in expected result set", expectedResults.hashedCentroids.contains(actualHashCentroid));
                actualCentroids.add(actualHashCentroid);
                totalResults++;
            } else {
                TestUtils.deleteAll(getDataStorePluginOptions());
                Assert.fail("Actual result '" + obj.toString() + "' is not of type Simple Feature.");
            }
        }
        for (final long l : actualCentroids) {
            expectedResults.hashedCentroids.remove(l);
        }
        for (final long l : expectedResults.hashedCentroids) {
            LOGGER.error("Missing expected hashed centroid: " + l);
        }
        if (expectedResults.count != totalResults) {
            TestUtils.deleteAll(getDataStorePluginOptions());
        }
        Assert.assertEquals(expectedResults.count, totalResults);
        final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
        long statisticsResult = 0;
        int duplicates = 0;
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
            AggregationQueryBuilder<?, Long, ?, ?> aggBldr = AggregationQueryBuilder.newBuilder();
            if (index != null) {
                aggBldr = aggBldr.indexName(index.getName());
            }
            aggBldr = aggBldr.constraints(constraints);
            if (countDuplicates) {
                aggBldr.aggregate(internalDataAdapter.getTypeName(), (Aggregation) new DuplicateCountAggregation());
                final DuplicateCount countResult = (DuplicateCount) geowaveStore.aggregate((AggregationQuery) aggBldr.build());
                if (countResult != null) {
                    duplicates += countResult.count;
                }
            }
            aggBldr.count(internalDataAdapter.getTypeName());
            final Long countResult = geowaveStore.aggregate(aggBldr.build());
            // results should already be aggregated, there should be
            // exactly one value in this iterator
            Assert.assertNotNull(countResult);
            statisticsResult += countResult;
        }
        Assert.assertEquals(expectedResults.count, statisticsResult - duplicates);
    }
}
Also used : AggregationQuery(org.locationtech.geowave.core.store.api.AggregationQuery) ArrayList(java.util.ArrayList) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) ExpectedResults(org.locationtech.geowave.test.TestUtils.ExpectedResults) SimpleFeature(org.opengis.feature.simple.SimpleFeature) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) DataStore(org.locationtech.geowave.core.store.api.DataStore) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter)

Aggregations

InternalDataAdapter (org.locationtech.geowave.core.store.adapter.InternalDataAdapter)31 PersistentAdapterStore (org.locationtech.geowave.core.store.adapter.PersistentAdapterStore)22 Index (org.locationtech.geowave.core.store.api.Index)19 QueryConstraints (org.locationtech.geowave.core.store.query.constraints.QueryConstraints)17 ArrayList (java.util.ArrayList)15 List (java.util.List)13 AdapterToIndexMapping (org.locationtech.geowave.core.store.AdapterToIndexMapping)13 AdapterIndexMappingStore (org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore)13 DataStore (org.locationtech.geowave.core.store.api.DataStore)13 IndexStore (org.locationtech.geowave.core.store.index.IndexStore)12 Map (java.util.Map)11 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)11 GeoWaveRow (org.locationtech.geowave.core.store.entities.GeoWaveRow)11 Pair (org.apache.commons.lang3.tuple.Pair)10 DataTypeAdapter (org.locationtech.geowave.core.store.api.DataTypeAdapter)10 Logger (org.slf4j.Logger)10 LoggerFactory (org.slf4j.LoggerFactory)10 Lists (com.google.common.collect.Lists)9 Arrays (java.util.Arrays)9 Entry (java.util.Map.Entry)9