Examples with BoundingBoxValue org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue used on opensource projects

Example 1 with BoundingBoxValue

use of org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue in project geowave by locationtech.

the class GeoWaveSparkIngestIT method testBasicSparkIngest.

@Test
public void testBasicSparkIngest() throws Exception {
    // ingest test points
    TestUtils.testSparkIngest(dataStore, DimensionalityType.SPATIAL, S3URL, GDELT_INPUT_FILES, "gdelt");
    final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        // query by the full bounding box, make sure there is more than
        // 0 count and make sure the count matches the number of results
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", new Integer(GDELT_COUNT), new Integer(resultCount));
    }
    // Clean up
    TestUtils.deleteAll(dataStore);
}

Example 2 with BoundingBoxValue

use of org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue in project geowave by locationtech.

the class BasicKafkaIT method testBasicIngestGpx.

@Test
public void testBasicIngestGpx() throws Exception {
    KafkaTestUtils.testKafkaStage(OSM_GPX_INPUT_DIR);
    KafkaTestUtils.testKafkaIngest(dataStorePluginOptions, false, OSM_GPX_INPUT_DIR);
    final DataStatisticsStore statsStore = dataStorePluginOptions.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStorePluginOptions.createAdapterStore();
    int adapterCount = 0;
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", EXPECTED_COUNT_PER_ADAPTER_ID.get(adapter.getTypeName()), new Integer(resultCount));
        adapterCount++;
    }
    assertTrue("There should be exactly two adapters", (adapterCount == 2));
}

Example 3 with BoundingBoxValue

use of org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}

Example 4 with BoundingBoxValue

use of org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue in project geowave by locationtech.

the class GeoWaveBasicURLIngestIT method testBasicURLIngest.

@Test
public void testBasicURLIngest() throws Exception {
    TestUtils.testS3LocalIngest(dataStore, DimensionalityType.SPATIAL, S3URL, GDELT_INPUT_FILE_URL, "gdelt", 4);
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final DataStore ds = dataStore.createDataStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        // query by the full bounding box, make sure there is more than
        // 0 count and make sure the count matches the number of results
        final BoundingBoxValue bbox = ds.aggregateStatistics(StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(internalDataAdapter.getTypeName()).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).build());
        assertNotNull(bbox);
        final CountValue count = ds.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(internalDataAdapter.getTypeName()).build());
        assertNotNull(count);
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bbox.getMinX(), bbox.getMaxX(), bbox.getMinY(), bbox.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", new Integer(GDELT_URL_COUNT), new Integer(resultCount));
    }
    // Clean up
    TestUtils.deleteAll(dataStore);
}

Example 5 with BoundingBoxValue

use of org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue in project geowave by locationtech.

the class GeoWaveStatisticsIT method testRecalcStatistic.

@Test
public void testRecalcStatistic() {
    final DataStore ds = dataStore.createDataStore();
    // Get bounding box statistic
    final Statistic<BoundingBoxValue> bboxStat = ds.getFieldStatistic(BoundingBoxStatistic.STATS_TYPE, SimpleIngest.FEATURE_NAME, SimpleIngest.GEOMETRY_FIELD, Statistic.INTERNAL_TAG);
    assertNotNull(bboxStat);
    // Get the value
    Envelope bbox = ds.getStatisticValue(bboxStat);
    assertEquals(-165.0, bbox.getMinX(), 0.1);
    assertEquals(180.0, bbox.getMaxX(), 0.1);
    assertEquals(-90.0, bbox.getMinY(), 0.1);
    assertEquals(85.0, bbox.getMaxY(), 0.1);
    // Delete half of the data
    final VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder();
    final Query<?> query = bldr.addTypeName(SimpleIngest.FEATURE_NAME).constraints(bldr.constraintsFactory().cqlConstraints("Longitude > 0")).build();
    assertTrue(ds.delete(query));
    // Verify the value was unchanged
    bbox = ds.getStatisticValue(bboxStat);
    assertEquals(-165.0, bbox.getMinX(), 0.1);
    assertEquals(180.0, bbox.getMaxX(), 0.1);
    assertEquals(-90.0, bbox.getMinY(), 0.1);
    assertEquals(85.0, bbox.getMaxY(), 0.1);
    // Recalculate the stat
    ds.recalcStatistic(bboxStat);
    // Verify the value was updated
    bbox = ds.getStatisticValue(bboxStat);
    assertEquals(-165.0, bbox.getMinX(), 0.1);
    assertEquals(0, bbox.getMaxX(), 0.1);
    assertEquals(-60.0, bbox.getMinY(), 0.1);
    assertEquals(80.0, bbox.getMaxY(), 0.1);
}