Examples with NumericRangeStatistic org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic used on opensource projects

Example 1 with NumericRangeStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic in project geowave by locationtech.

the class GeoWaveStatisticsIT method testAddStatistic.

@Test
public void testAddStatistic() {
    final DataStore ds = dataStore.createDataStore();
    final NumericRangeStatistic longitudeRange = new NumericRangeStatistic(SimpleIngest.FEATURE_NAME, "Longitude");
    final NumericRangeStatistic latitudeRange = new NumericRangeStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    final TimeRangeStatistic timeRange = new TimeRangeStatistic(SimpleIngest.FEATURE_NAME, "TimeStamp");
    final NumericStatsStatistic latitudeStats = new NumericStatsStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    final BloomFilterStatistic latitudeBloomFilter = new BloomFilterStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    final NumericHistogramStatistic latitudeHistogram = new NumericHistogramStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    ds.addStatistic(longitudeRange, timeRange, latitudeStats, latitudeBloomFilter, latitudeHistogram);
    ds.addEmptyStatistic(latitudeRange);
    try (CloseableIterator<NumericRangeValue> iterator = ds.queryStatistics(StatisticQueryBuilder.newBuilder(NumericRangeStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).fieldName("Longitude").build())) {
        assertTrue(iterator.hasNext());
        final NumericRangeValue value = iterator.next();
        assertEquals(-165.0, value.getMin(), 0.1);
        assertEquals(180.0, value.getMax(), 0.1);
        assertFalse(iterator.hasNext());
    }
    try (CloseableIterator<NumericRangeValue> iterator = ds.queryStatistics(StatisticQueryBuilder.newBuilder(NumericRangeStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).fieldName("Latitude").build())) {
        // We only calculated stats for Longitude
        assertTrue(iterator.hasNext());
        assertFalse(iterator.next().isSet());
        assertFalse(iterator.hasNext());
    }
    final Interval interval = ds.getStatisticValue(timeRange);
    try (CloseableIterator<SimpleFeature> it = ds.query(VectorQueryBuilder.newBuilder().build())) {
        long min = Long.MAX_VALUE, max = Long.MIN_VALUE;
        while (it.hasNext()) {
            final long time = ((Date) it.next().getAttribute("TimeStamp")).getTime();
            min = Math.min(min, time);
            max = Math.max(max, time);
        }
        assertEquals(min, interval.getStart().toEpochMilli());
        assertEquals(max, interval.getEnd().toEpochMilli());
    }
    final Stats stats = ds.getStatisticValue(latitudeStats);
    assertEquals(20L, stats.count());
    assertEquals(-90.0, stats.min(), 0.1);
    assertEquals(85.0, stats.max(), 0.1);
    assertEquals(-0.5, stats.mean(), 0.1);
    assertEquals(53.47, stats.populationStandardDeviation(), 0.1);
    final BloomFilter<CharSequence> bloomFilter = ds.getStatisticValue(latitudeBloomFilter);
    boolean expectLat = true;
    for (double lat = -90; lat <= 90; lat += 5) {
        if (expectLat) {
            assertTrue(bloomFilter.mightContain(Double.toString(lat)));
        } else {
            assertFalse(bloomFilter.mightContain(Double.toString(lat)));
        }
        // and forth each iteration, 3 times it stays true at these latitudes
        if (!DoubleMath.fuzzyEquals(-40, lat, 0.1) && !DoubleMath.fuzzyEquals(25, lat, 0.1) && !DoubleMath.fuzzyEquals(80, lat, 0.1)) {
            expectLat = !expectLat;
        }
    }
    final NumericHistogram histogram = ds.getStatisticValue(latitudeHistogram);
    assertEquals(20L, histogram.getTotalCount(), 0.1);
    assertEquals(-90.0, histogram.getMinValue(), 0.1);
    assertEquals(85.0, histogram.getMaxValue(), 0.1);
    assertEquals(0.0, histogram.quantile(0.5), 0.1);
}

Example 2 with NumericRangeStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic in project geowave by locationtech.

the class GeoWaveGTPluginUtils method accepts.

protected static boolean accepts(final DataStatisticsStore statisticsStore, final DataTypeAdapter<?> adapter, final org.opengis.feature.FeatureVisitor visitor, final org.opengis.util.ProgressListener progress, final SimpleFeatureType featureType) throws IOException {
    if ((visitor instanceof MinVisitor)) {
        final ExtractAttributesFilter filter = new ExtractAttributesFilter();
        final MinVisitor minVisitor = (MinVisitor) visitor;
        final Collection<String> attrs = (Collection<String>) minVisitor.getExpression().accept(filter, null);
        int acceptedCount = 0;
        final Map<String, List<FieldStatistic<?>>> adapterFieldStatistics = getFieldStats(statisticsStore, adapter);
        for (final String attr : attrs) {
            if (!adapterFieldStatistics.containsKey(attr)) {
                continue;
            }
            for (final FieldStatistic<?> stat : adapterFieldStatistics.get(attr)) {
                if ((stat instanceof TimeRangeStatistic) && (stat.getBinningStrategy() == null)) {
                    final TimeRangeValue statValue = statisticsStore.getStatisticValue((TimeRangeStatistic) stat);
                    if (statValue != null) {
                        minVisitor.setValue(convertToType(attr, new Date(statValue.getMin()), featureType));
                        acceptedCount++;
                    }
                } else if (stat instanceof NumericRangeStatistic) {
                    try (CloseableIterator<NumericRangeValue> values = statisticsStore.getStatisticValues((NumericRangeStatistic) stat)) {
                        NumericRangeValue statValue = ((NumericRangeStatistic) stat).createEmpty();
                        while (values.hasNext()) {
                            statValue.merge(values.next());
                        }
                        if (statValue.isSet()) {
                            minVisitor.setValue(convertToType(attr, statValue.getMin(), featureType));
                            acceptedCount++;
                        }
                    }
                }
            }
        }
        if (acceptedCount > 0) {
            if (progress != null) {
                progress.complete();
            }
            return true;
        }
    } else if ((visitor instanceof MaxVisitor)) {
        final ExtractAttributesFilter filter = new ExtractAttributesFilter();
        final MaxVisitor maxVisitor = (MaxVisitor) visitor;
        final Collection<String> attrs = (Collection<String>) maxVisitor.getExpression().accept(filter, null);
        int acceptedCount = 0;
        final Map<String, List<FieldStatistic<?>>> adapterFieldStatistics = getFieldStats(statisticsStore, adapter);
        for (final String attr : attrs) {
            for (final FieldStatistic<?> stat : adapterFieldStatistics.get(attr)) {
                if ((stat instanceof TimeRangeStatistic) && (stat.getBinningStrategy() == null)) {
                    final TimeRangeValue statValue = statisticsStore.getStatisticValue((TimeRangeStatistic) stat);
                    if (statValue != null) {
                        maxVisitor.setValue(convertToType(attr, new Date(statValue.getMax()), featureType));
                        acceptedCount++;
                    }
                } else if (stat instanceof NumericRangeStatistic) {
                    try (CloseableIterator<NumericRangeValue> values = statisticsStore.getStatisticValues((NumericRangeStatistic) stat)) {
                        NumericRangeValue statValue = ((NumericRangeStatistic) stat).createEmpty();
                        while (values.hasNext()) {
                            statValue.merge(values.next());
                        }
                        if (statValue.isSet()) {
                            maxVisitor.setValue(convertToType(attr, statValue.getMax(), featureType));
                            acceptedCount++;
                        }
                    }
                }
            }
        }
        if (acceptedCount > 0) {
            if (progress != null) {
                progress.complete();
            }
            return true;
        }
    }
    return false;
}

Example 3 with NumericRangeStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic in project geowave by locationtech.

the class GeoWaveFeatureSourceTest method testFull.

public void testFull(final Populater populater, final String ext) throws Exception {
    final String typeName = "GeoWaveFeatureSourceTest_full" + ext;
    final SimpleFeatureType type = DataUtilities.createType(typeName, "geometry:Geometry:srid=4326,pop:java.lang.Long,pid:String,when:Date");
    final DataStore dataStore = createDataStore();
    final GeoWaveGTDataStore gwgtDataStore = (GeoWaveGTDataStore) dataStore;
    gwgtDataStore.dataStatisticsStore.addStatistic(new NumericRangeStatistic(typeName, "pop"));
    populater.populate(type, dataStore);
    final SimpleFeatureSource source = dataStore.getFeatureSource(typeName);
    final ReferencedEnvelope env = source.getBounds();
    assertEquals(43.454, env.getMaxX(), 0.0001);
    assertEquals(27.232, env.getMinY(), 0.0001);
    assertEquals(28.242, env.getMaxY(), 0.0001);
    final Query query = new Query(typeName, Filter.INCLUDE);
    assertTrue(source.getCount(query) > 2);
    final short internalAdapterId = ((GeoWaveGTDataStore) dataStore).getInternalAdapterStore().addTypeName(typeName);
    final DataStatisticsStore statsStore = ((GeoWaveGTDataStore) dataStore).getDataStatisticsStore();
    final DataTypeAdapter<?> adapter = ((GeoWaveGTDataStore) dataStore).getAdapterStore().getAdapter(internalAdapterId);
    BoundingBoxValue bboxStats = null;
    CountValue cStats = null;
    TimeRangeValue timeRangeStats = null;
    NumericRangeValue popStats = null;
    int count = 1;
    cStats = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, typeName);
    assertNotNull(cStats);
    try (final CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getFieldStatistics(adapter, null, null, null)) {
        assertTrue(stats.hasNext());
        while (stats.hasNext()) {
            final Statistic<?> stat = stats.next();
            if (stat instanceof BoundingBoxStatistic) {
                bboxStats = statsStore.getStatisticValue((BoundingBoxStatistic) stat);
            } else if (stat instanceof TimeRangeStatistic) {
                timeRangeStats = statsStore.getStatisticValue((TimeRangeStatistic) stat);
            } else if (stat instanceof NumericRangeStatistic) {
                popStats = statsStore.getStatisticValue((NumericRangeStatistic) stat);
            }
            count++;
        }
    }
    // rather than maintain an exact count on stats as we should be able
    // to add them more dynamically, just make sure that there is some
    // set of base stats found
    assertTrue("Unexpectedly few stats found", count >= 4);
    assertEquals(66, popStats.getMin(), 0.001);
    assertEquals(100, popStats.getMax(), 0.001);
    assertEquals(DateUtilities.parseISO("2005-05-17T20:32:56Z"), timeRangeStats.asTemporalRange().getStartTime());
    assertEquals(DateUtilities.parseISO("2005-05-19T20:32:56Z"), timeRangeStats.asTemporalRange().getEndTime());
    assertEquals(43.454, bboxStats.getMaxX(), 0.0001);
    assertEquals(27.232, bboxStats.getMinY(), 0.0001);
    assertEquals(3, (long) cStats.getValue());
}

Example 4 with NumericRangeStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic in project geowave by locationtech.

the class WFSTransactionTest method setup.

@Before
public void setup() throws SchemaException, CQLException, IOException, GeoWavePluginException {
    dataStore = createDataStore();
    type = DataUtilities.createType("geostuff", "geometry:Geometry:srid=4326,pop:java.lang.Long,pid:String");
    dataStore.createSchema(type);
    query = new Query("geostuff", CQL.toFilter("BBOX(geometry,27.20,41.20,27.30,41.30)"), new String[] { "geometry", "pid" });
    if (dataStore instanceof GeoWaveGTDataStore) {
        ((GeoWaveGTDataStore) dataStore).dataStore.addEmptyStatistic(new NumericRangeStatistic(type.getTypeName(), "pop"));
    }
}

Example 5 with NumericRangeStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic in project geowave by locationtech.

the class MemoryDataStoreTest method testMultipleIndices.

@Test
public void testMultipleIndices() throws IOException {
    final Index index1 = new IndexImpl(new MockComponents.MockIndexStrategy(), new MockComponents.TestIndexModel("tm1"));
    final Index index2 = new IndexImpl(new MockComponents.MockIndexStrategy(), new MockComponents.TestIndexModel("tm2"));
    final String namespace = "test2_" + getClass().getName();
    final StoreFactoryFamilySpi storeFamily = new MemoryStoreFactoryFamily();
    final MemoryRequiredOptions opts = new MemoryRequiredOptions();
    opts.setGeoWaveNamespace(namespace);
    final DataStore dataStore = storeFamily.getDataStoreFactory().createStore(opts);
    final DataStatisticsStore statsStore = storeFamily.getDataStatisticsStoreFactory().createStore(opts);
    final DataTypeAdapter<Integer> adapter = new MockComponents.MockAbstractDataAdapter();
    final VisibilityHandler visHandler = new GlobalVisibilityHandler("aaa&bbb");
    final List<Statistic<?>> statistics = Lists.newArrayList();
    statistics.add(new NumericRangeStatistic(adapter.getTypeName(), MockAbstractDataAdapter.INTEGER));
    dataStore.addType(adapter, statistics, index1, index2);
    try (final Writer<Integer> indexWriter = dataStore.createWriter(adapter.getTypeName())) {
        indexWriter.write(new Integer(25), visHandler);
        indexWriter.flush();
        indexWriter.write(new Integer(35), visHandler);
        indexWriter.flush();
    }
    // authorization check
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index2.getName()).addAuthorization("aaa").constraints(new TestQuery(23, 26)).build())) {
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index1.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 26)).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(25), itemIt.next());
        assertFalse(itemIt.hasNext());
    }
    // pick an index
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 36)).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(25), itemIt.next());
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIt = statsStore.getAllStatistics(null)) {
        try (CloseableIterator<? extends StatisticValue<?>> statisticValues = statsStore.getStatisticValues(statsIt, null, "aaa", "bbb")) {
            assertTrue(checkStats(statisticValues, 2, new NumericRange(25, 35)));
        }
    }
    try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIt = statsStore.getAllStatistics(null)) {
        try (CloseableIterator<? extends StatisticValue<?>> statisticValues = statsStore.getStatisticValues(statsIt, null)) {
            assertTrue(checkStats(statisticValues, 0, null));
        }
    }
    dataStore.delete(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 26)).build());
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index1.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 36)).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index2.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 36)).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index1.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 26)).build())) {
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index2.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 26)).build())) {
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index1.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new DataIdQuery(adapter.getDataId(new Integer(35)))).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index2.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new DataIdQuery(adapter.getDataId(new Integer(35)))).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
    }
}