Search in sources :

Example 1 with GeoToolsVectorDataStoreIngestPlugin

use of org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}
Also used : FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Arrays(java.util.Arrays) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) URL(java.net.URL) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) CommonIndexAggregation(org.locationtech.geowave.core.store.query.aggregate.CommonIndexAggregation) LoggerFactory(org.slf4j.LoggerFactory) Aggregation(org.locationtech.geowave.core.store.api.Aggregation) MathUtils(org.apache.commons.math.util.MathUtils) TestUtils(org.locationtech.geowave.test.TestUtils) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) ByteBuffer(java.nio.ByteBuffer) TimeDescriptors(org.locationtech.geowave.core.geotime.util.TimeDescriptors) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore) StatisticValue(org.locationtech.geowave.core.store.api.StatisticValue) Pair(org.apache.commons.lang3.tuple.Pair) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Map(java.util.Map) Statistic(org.locationtech.geowave.core.store.api.Statistic) Maps(jersey.repackaged.com.google.common.collect.Maps) Persistable(org.locationtech.geowave.core.index.persist.Persistable) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FieldStatistic(org.locationtech.geowave.core.store.api.FieldStatistic) StatisticQuery(org.locationtech.geowave.core.store.api.StatisticQuery) Set(java.util.Set) ManualOperationParams(org.locationtech.geowave.core.cli.parser.ManualOperationParams) DimensionalityType(org.locationtech.geowave.test.TestUtils.DimensionalityType) ExpectedResults(org.locationtech.geowave.test.TestUtils.ExpectedResults) ConfigOptions(org.locationtech.geowave.core.cli.operations.config.options.ConfigOptions) List(java.util.List) VectorLocalExportOptions(org.locationtech.geowave.adapter.vector.export.VectorLocalExportOptions) Entry(java.util.Map.Entry) DataIdQuery(org.locationtech.geowave.core.store.query.constraints.DataIdQuery) Geometry(org.locationtech.jts.geom.Geometry) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) CoordinateReferenceSystem(org.opengis.referencing.crs.CoordinateReferenceSystem) ByteArray(org.locationtech.geowave.core.index.ByteArray) AggregationQuery(org.locationtech.geowave.core.store.api.AggregationQuery) BeforeClass(org.junit.BeforeClass) AddStoreCommand(org.locationtech.geowave.core.store.cli.store.AddStoreCommand) AggregationQueryBuilder(org.locationtech.geowave.core.store.api.AggregationQueryBuilder) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) CommonIndexedPersistenceEncoding(org.locationtech.geowave.core.store.data.CommonIndexedPersistenceEncoding) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) LocalFileIngestPlugin(org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin) Calendar(java.util.Calendar) Lists(com.google.common.collect.Lists) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) StatisticQueryBuilder(org.locationtech.geowave.core.store.api.StatisticQueryBuilder) QueryBuilder(org.locationtech.geowave.core.store.api.QueryBuilder) Index(org.locationtech.geowave.core.store.api.Index) StatisticsIngestCallback(org.locationtech.geowave.core.store.statistics.StatisticsIngestCallback) BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) GeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.GeotoolsFeatureDataAdapter) Logger(org.slf4j.Logger) DataStore(org.locationtech.geowave.core.store.api.DataStore) OptimalCQLQuery(org.locationtech.geowave.core.geotime.store.query.OptimalCQLQuery) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) IngestCallback(org.locationtech.geowave.core.store.callback.IngestCallback) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) VectorLocalExportCommand(org.locationtech.geowave.adapter.vector.export.VectorLocalExportCommand) File(java.io.File) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) InternalGeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.InternalGeotoolsFeatureDataAdapter) MemoryAdapterStore(org.locationtech.geowave.core.store.memory.MemoryAdapterStore) Filter(org.opengis.filter.Filter) ZipUtils(org.locationtech.geowave.adapter.raster.util.ZipUtils) Assert(org.junit.Assert) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) Envelope(org.locationtech.jts.geom.Envelope) Assert.assertEquals(org.junit.Assert.assertEquals) StatisticValue(org.locationtech.geowave.core.store.api.StatisticValue) HashMap(java.util.HashMap) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) Envelope(org.locationtech.jts.geom.Envelope) URL(java.net.URL) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) Entry(java.util.Map.Entry) Statistic(org.locationtech.geowave.core.store.api.Statistic) FieldStatistic(org.locationtech.geowave.core.store.api.FieldStatistic) BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) ByteArray(org.locationtech.geowave.core.index.ByteArray) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore) MemoryAdapterStore(org.locationtech.geowave.core.store.memory.MemoryAdapterStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) GeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.GeotoolsFeatureDataAdapter) InternalGeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.InternalGeotoolsFeatureDataAdapter) Map(java.util.Map) HashMap(java.util.HashMap)

Example 2 with GeoToolsVectorDataStoreIngestPlugin

use of org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin in project geowave by locationtech.

the class GeoWaveSpatialBinningStatisticsIT method testLineGeometry.

@SuppressWarnings({ "unchecked", "rawtypes" })
@Test
public void testLineGeometry() throws MalformedURLException, IOException {
    final DataStore store = dataStoreOptions.createDataStore();
    new GeoToolsVectorDataOptions();
    store.ingest(TORNADO_TRACKS_SHAPEFILE_FILE, IngestOptions.newBuilder().threads(4).format((LocalFileIngestPlugin) new GeoToolsVectorDataStoreIngestPlugin(new StringToIntRetypingPlugin())).build(), DimensionalityType.SPATIAL_TEMPORAL.getDefaultIndices());
    final SimpleFeatureType featureType = ((FeatureDataAdapter) store.getTypes()[0]).getFeatureType();
    testGeometry(featureType, store);
    testNumericStat(featureType, store);
}
Also used : GeoToolsVectorDataOptions(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataOptions) SimpleFeatureType(org.opengis.feature.simple.SimpleFeatureType) DataStore(org.locationtech.geowave.core.store.api.DataStore) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Test(org.junit.Test)

Aggregations

FeatureDataAdapter (org.locationtech.geowave.adapter.vector.FeatureDataAdapter)2 Lists (com.google.common.collect.Lists)1 File (java.io.File)1 IOException (java.io.IOException)1 URISyntaxException (java.net.URISyntaxException)1 URL (java.net.URL)1 ByteBuffer (java.nio.ByteBuffer)1 SimpleDateFormat (java.text.SimpleDateFormat)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Calendar (java.util.Calendar)1 Date (java.util.Date)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Set (java.util.Set)1 Maps (jersey.repackaged.com.google.common.collect.Maps)1 FileUtils (org.apache.commons.io.FileUtils)1