Search in sources :

Example 1 with CountStatistic

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic in project geowave by locationtech.

the class FeatureDataAdapter method getDefaultStatistics.

@Override
public List<Statistic<? extends StatisticValue<?>>> getDefaultStatistics() {
    final List<Statistic<?>> statistics = Lists.newArrayList();
    final CountStatistic count = new CountStatistic(getTypeName());
    count.setInternal();
    statistics.add(count);
    for (int i = 0; i < featureType.getAttributeCount(); i++) {
        final AttributeDescriptor ad = featureType.getDescriptor(i);
        if (Geometry.class.isAssignableFrom(ad.getType().getBinding())) {
            final BoundingBoxStatistic bbox = new BoundingBoxStatistic(getTypeName(), ad.getLocalName());
            bbox.setInternal();
            statistics.add(bbox);
        }
    }
    final TimeDescriptors timeDescriptors = getTimeDescriptors();
    if (timeDescriptors.hasTime()) {
        if (timeDescriptors.getTime() != null) {
            final TimeRangeStatistic timeRange = new TimeRangeStatistic(getTypeName(), timeDescriptors.getTime().getLocalName());
            timeRange.setInternal();
            statistics.add(timeRange);
        }
        if (timeDescriptors.getStartRange() != null) {
            final TimeRangeStatistic timeRange = new TimeRangeStatistic(getTypeName(), timeDescriptors.getStartRange().getLocalName());
            timeRange.setInternal();
            statistics.add(timeRange);
        }
        if (timeDescriptors.getEndRange() != null) {
            final TimeRangeStatistic timeRange = new TimeRangeStatistic(getTypeName(), timeDescriptors.getEndRange().getLocalName());
            timeRange.setInternal();
            statistics.add(timeRange);
        }
    }
    return statistics;
}
Also used : BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) TimeDescriptors(org.locationtech.geowave.core.geotime.util.TimeDescriptors) Statistic(org.locationtech.geowave.core.store.api.Statistic) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) TimeRangeStatistic(org.locationtech.geowave.core.geotime.store.statistics.TimeRangeStatistic) AttributeDescriptor(org.opengis.feature.type.AttributeDescriptor) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) TimeRangeStatistic(org.locationtech.geowave.core.geotime.store.statistics.TimeRangeStatistic)

Example 2 with CountStatistic

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic in project geowave by locationtech.

the class SpatialBinningStatisticExample method main.

public static void main(final String[] args) {
    final SimpleFeatureType featureType = getSimpleFeatureType();
    // Points (to be ingested into GeoWave Data Store)
    final List<SimpleFeature> cannedFeatures = ImmutableList.of(buildSimpleFeature(featureType, "Loc1", new Coordinate(-77.0352, 38.8895), 12), buildSimpleFeature(featureType, "Loc2", new Coordinate(-77.0366, 38.8977), 13), buildSimpleFeature(featureType, "Loc3", new Coordinate(-76.8644, 38.9078), 8), buildSimpleFeature(featureType, "Loc4", new Coordinate(-76.350677, 38.9641511), 15), buildSimpleFeature(featureType, "Loc5", new Coordinate(-77.3384112, 38.416091), 7), buildSimpleFeature(featureType, "Loc6", new Coordinate(-67.0352, 28.8895), 3), buildSimpleFeature(featureType, "Loc7", new Coordinate(-67.0366, 28.8977), 99), buildSimpleFeature(featureType, "Loc8", new Coordinate(-66.8644, 28.9078), 0), buildSimpleFeature(featureType, "Loc9", new Coordinate(-66.350677, 28.9641511), 1), buildSimpleFeature(featureType, "Loc10", new Coordinate(-67.3384112, 28.416091), 23));
    final Index index = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final DataStore dataStore = DataStoreFactory.createDataStore(new MemoryRequiredOptions());
    final FeatureDataAdapter adapter = new FeatureDataAdapter(featureType);
    final Envelope bbox1 = new Envelope(-77.5, -76, 38.4, 39);
    final Envelope bbox2 = new Envelope(-67.5, -66, 28.4, 29);
    dataStore.addType(adapter, index);
    final CountStatistic s2Count = new CountStatistic(featureType.getTypeName());
    s2Count.setTag("S2-Example");
    final SpatialFieldValueBinningStrategy s2SpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    // type could be Google's S2, Uber's H3, or simple GeoHash
    s2SpatialBinning.setType(SpatialBinningType.S2);
    // precision is the character length for H3 and GeoHash which is over twice as coarse as S2
    // which uses powers of two for precision (so a precision of 8 in S2 is actually a coarser
    // granularity than a precision of 4 in GeoHash or H3)
    s2SpatialBinning.setPrecision(7);
    s2Count.setBinningStrategy(s2SpatialBinning);
    final CountStatistic h3Count = new CountStatistic(featureType.getTypeName());
    // stats for the same feature type should have different tags
    h3Count.setTag("H3-Example");
    final SpatialFieldValueBinningStrategy h3SpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    // type could be Google's S2, Uber's H3, or simple GeoHash
    h3SpatialBinning.setType(SpatialBinningType.H3);
    h3SpatialBinning.setPrecision(3);
    h3Count.setBinningStrategy(h3SpatialBinning);
    final CountStatistic geohashCount = new CountStatistic(featureType.getTypeName());
    geohashCount.setTag("Geohash-Example");
    final SpatialFieldValueBinningStrategy geohashSpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    // type could be Google's S2, Uber's H3, or simple GeoHash
    geohashSpatialBinning.setType(SpatialBinningType.GEOHASH);
    geohashSpatialBinning.setPrecision(3);
    geohashCount.setBinningStrategy(geohashSpatialBinning);
    // you can add "empty" statistic before you've written any data, the stats will then be updated
    // as you write data
    // alternatively if you don't use the "empty" variant it will automatically calculate and update
    // these stats for pre-existing data before returning from the method
    dataStore.addEmptyStatistic(s2Count, h3Count, geohashCount);
    // Ingest cannedFeatures into the DataStore.
    try (Writer<SimpleFeature> indexWriter = dataStore.createWriter(adapter.getTypeName())) {
        for (final SimpleFeature sf : cannedFeatures) {
            indexWriter.write(sf);
        }
    }
    System.out.println("***** S2 Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(s2Count)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), s2SpatialBinning.binToString(pair.getLeft()), s2SpatialBinning.getType().getBinGeometry(pair.getLeft(), 7)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(s2Count, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), s2SpatialBinning.binToString(pair.getLeft()), s2SpatialBinning.getType().getBinGeometry(pair.getLeft(), 7)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** %d in bbox %s **", dataStore.getStatisticValue(s2Count, BinConstraints.ofObject(bbox2)), bbox2));
    System.out.println("\n***** H3 Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(h3Count)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), h3SpatialBinning.binToString(pair.getLeft()), h3SpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(h3Count, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), h3SpatialBinning.binToString(pair.getLeft()), h3SpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** %d in bbox %s **", dataStore.getStatisticValue(h3Count, BinConstraints.ofObject(bbox2)), bbox2));
    System.out.println("\n***** Geohash Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(geohashCount)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), geohashSpatialBinning.binToString(pair.getLeft()), geohashSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(geohashCount, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), geohashSpatialBinning.binToString(pair.getLeft()), geohashSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** %d in bbox %s **", dataStore.getStatisticValue(geohashCount, BinConstraints.ofObject(bbox2)), bbox2));
    // and finally just to make it clear, you can apply spatial binning to *any* statistic not just
    // counts
    // so here's an example binning numeric stats of the population (sum, avg, std dev, etc.) by an
    // S2 level 7 grid
    final NumericStatsStatistic s2PopulationStats = new NumericStatsStatistic(featureType.getTypeName(), "population");
    s2PopulationStats.setTag("S2-Population-Stats");
    final SpatialFieldValueBinningStrategy s2PopulationSpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    s2PopulationSpatialBinning.setType(SpatialBinningType.S2);
    s2PopulationSpatialBinning.setPrecision(7);
    s2PopulationStats.setBinningStrategy(s2PopulationSpatialBinning);
    // here we'll calculate the stat on add based on the already written data (rather than adding
    // the "empty" statistic)
    dataStore.addStatistic(s2PopulationStats);
    // and we'll run through the same set of examples of getting all the bins and then filtering by
    // an envelope
    System.out.println("\n***** S2 Population Stats Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Stats>> it = dataStore.getBinnedStatisticValues(s2PopulationStats)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Stats> pair = it.next();
            System.out.println(String.format("Population: %s, Bin: %s, Bin Geometry: %s", pair.getRight(), s2PopulationSpatialBinning.binToString(pair.getLeft()), s2PopulationSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Stats>> it = dataStore.getBinnedStatisticValues(s2PopulationStats, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Stats> pair = it.next();
            System.out.println(String.format("Population: %s, Bin: %s, Bin Geometry: %s", pair.getRight(), s2PopulationSpatialBinning.binToString(pair.getLeft()), s2PopulationSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** Population Stats '%s' in bbox %s **", dataStore.getStatisticValue(s2PopulationStats, BinConstraints.ofObject(bbox2)), bbox2));
}
Also used : SpatialFieldValueBinningStrategy(org.locationtech.geowave.core.geotime.store.statistics.binning.SpatialFieldValueBinningStrategy) NumericStatsStatistic(org.locationtech.geowave.core.store.statistics.field.NumericStatsStatistic) Index(org.locationtech.geowave.core.store.api.Index) Envelope(org.locationtech.jts.geom.Envelope) SpatialOptions(org.locationtech.geowave.core.geotime.index.SpatialOptions) SimpleFeature(org.opengis.feature.simple.SimpleFeature) SimpleFeatureType(org.opengis.feature.simple.SimpleFeatureType) Coordinate(org.locationtech.jts.geom.Coordinate) DataStore(org.locationtech.geowave.core.store.api.DataStore) Stats(org.locationtech.geowave.core.store.statistics.field.Stats) ByteArray(org.locationtech.geowave.core.index.ByteArray) MemoryRequiredOptions(org.locationtech.geowave.core.store.memory.MemoryRequiredOptions) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) Pair(org.apache.commons.lang3.tuple.Pair)

Example 3 with CountStatistic

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic in project geowave by locationtech.

the class GeoWaveVisibilityIT method testMixedVisibilityStatistics.

public static void testMixedVisibilityStatistics(final DataStorePluginOptions dataStoreOptions, final VisibilityHandler visibilityHandler, final int totalFeatures) {
    final SimpleFeatureBuilder bldr = new SimpleFeatureBuilder(getType());
    final FeatureDataAdapter adapter = new FeatureDataAdapter(getType());
    final DataStore store = dataStoreOptions.createDataStore();
    // Add some statistics
    final CountStatistic geomCount = new CountStatistic();
    geomCount.setTag("testGeom");
    geomCount.setTypeName(adapter.getTypeName());
    geomCount.setBinningStrategy(new FieldValueBinningStrategy("geometry"));
    final CountStatistic visCountC = new CountStatistic();
    visCountC.setTag("testC");
    visCountC.setTypeName(adapter.getTypeName());
    visCountC.setBinningStrategy(new FieldValueBinningStrategy("c"));
    final CountStatistic visCountAB = new CountStatistic();
    visCountAB.setTag("testAB");
    visCountAB.setTypeName(adapter.getTypeName());
    visCountAB.setBinningStrategy(new FieldValueBinningStrategy("a", "b"));
    // Specify visibility at the type level
    store.addType(adapter, visibilityHandler, Lists.newArrayList(geomCount, visCountC, visCountAB), TestUtils.DEFAULT_SPATIAL_INDEX);
    try (Writer<SimpleFeature> writer = store.createWriter(adapter.getTypeName())) {
        for (int i = 0; i < totalFeatures; i++) {
            bldr.set("a", A_FIELD_VALUES[i % 3]);
            bldr.set("b", B_FIELD_VALUES[i % 3]);
            bldr.set("c", C_FIELD_VALUES[i % 3]);
            bldr.set("geometry", new GeometryFactory().createPoint(new Coordinate(0, 0)));
            writer.write(bldr.buildFeature(Integer.toString(i)));
        }
    }
    // Since each field is only visible if you provide that field ID as an authorization, each
    // statistic should only reveal those counts if the appropriate authorization is set. Because
    // these statistics are using a field value binning strategy, the actual bins of the statistic
    // may reveal information that is not authorized to the user and should be hidden.
    final CountValue countCNoAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testC").build());
    assertEquals(0, countCNoAuth.getValue().longValue());
    // When providing the "c" auth, all values should be present
    final CountValue countCAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testC").addAuthorization("c").build());
    assertEquals(totalFeatures, countCAuth.getValue().longValue());
    // For the AB count statistic, the values should only be present if both "a" and "b"
    // authorizations are provided
    final CountValue countABNoAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").build());
    assertEquals(0, countABNoAuth.getValue().longValue());
    final CountValue countABOnlyAAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("a").build());
    assertEquals(0, countABOnlyAAuth.getValue().longValue());
    final CountValue countABOnlyBAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("b").build());
    assertEquals(0, countABOnlyBAuth.getValue().longValue());
    final CountValue countABAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("a").addAuthorization("b").build());
    assertEquals(totalFeatures, countABAuth.getValue().longValue());
    // It should also work if additional authorizations are provided
    final CountValue countABCAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("a").addAuthorization("b").addAuthorization("c").build());
    assertEquals(totalFeatures, countABCAuth.getValue().longValue());
    // Since the geometry field has no visibility, no authorizations should be required
    final CountValue countGeomNoAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testGeom").build());
    assertEquals(totalFeatures, countGeomNoAuth.getValue().longValue());
}
Also used : FieldValueBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.FieldValueBinningStrategy) GeometryFactory(org.locationtech.jts.geom.GeometryFactory) Coordinate(org.locationtech.jts.geom.Coordinate) DifferingVisibilityCountValue(org.locationtech.geowave.core.store.statistics.index.DifferingVisibilityCountStatistic.DifferingVisibilityCountValue) CountValue(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue) DataStore(org.locationtech.geowave.core.store.api.DataStore) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) DifferingVisibilityCountStatistic(org.locationtech.geowave.core.store.statistics.index.DifferingVisibilityCountStatistic) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Point(org.locationtech.jts.geom.Point) SimpleFeatureBuilder(org.geotools.feature.simple.SimpleFeatureBuilder)

Example 4 with CountStatistic

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic in project geowave by locationtech.

the class MemoryDataStoreTest method test.

@Test
public void test() throws IOException {
    final Index index = new IndexImpl(new MockComponents.MockIndexStrategy(), new MockComponents.TestIndexModel());
    final String namespace = "test_" + getClass().getName();
    final StoreFactoryFamilySpi storeFamily = new MemoryStoreFactoryFamily();
    final MemoryRequiredOptions reqOptions = new MemoryRequiredOptions();
    reqOptions.setGeoWaveNamespace(namespace);
    final DataStore dataStore = storeFamily.getDataStoreFactory().createStore(reqOptions);
    final DataStatisticsStore statsStore = storeFamily.getDataStatisticsStoreFactory().createStore(reqOptions);
    final DataTypeAdapter<Integer> adapter = new MockComponents.MockAbstractDataAdapter();
    final VisibilityHandler visHandler = new GlobalVisibilityHandler("aaa&bbb");
    final List<Statistic<?>> statistics = Lists.newArrayList();
    statistics.add(new CountStatistic(adapter.getTypeName()));
    statistics.add(new NumericRangeStatistic(adapter.getTypeName(), MockAbstractDataAdapter.INTEGER));
    dataStore.addType(adapter, statistics, index);
    try (final Writer<Integer> indexWriter = dataStore.createWriter(adapter.getTypeName())) {
        indexWriter.write(new Integer(25), visHandler);
        indexWriter.flush();
        indexWriter.write(new Integer(35), visHandler);
        indexWriter.flush();
    }
    // authorization check
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName()).addAuthorization("aaa").constraints(new TestQuery(23, 26)).build())) {
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 26)).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(25), itemIt.next());
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 36)).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(25), itemIt.next());
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIt = statsStore.getAllStatistics(null)) {
        try (CloseableIterator<? extends StatisticValue<?>> statisticValues = statsStore.getStatisticValues(statsIt, null, "aaa", "bbb")) {
            assertTrue(checkStats(statisticValues, 2, new NumericRange(25, 35)));
        }
    }
    try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIt = statsStore.getAllStatistics(null)) {
        try (CloseableIterator<? extends StatisticValue<?>> statisticValues = statsStore.getStatisticValues(statsIt, null)) {
            assertTrue(checkStats(statisticValues, 0, null));
        }
    }
    dataStore.delete(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 26)).build());
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 36)).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new TestQuery(23, 26)).build())) {
        assertFalse(itemIt.hasNext());
    }
    try (CloseableIterator<?> itemIt = dataStore.query(QueryBuilder.newBuilder().addTypeName(adapter.getTypeName()).indexName(index.getName()).addAuthorization("aaa").addAuthorization("bbb").constraints(new DataIdQuery(adapter.getDataId(new Integer(35)))).build())) {
        assertTrue(itemIt.hasNext());
        assertEquals(new Integer(35), itemIt.next());
    }
}
Also used : NumericRangeStatistic(org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic) Index(org.locationtech.geowave.core.store.api.Index) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) NumericRange(org.locationtech.geowave.core.index.numeric.NumericRange) MockAbstractDataAdapter(org.locationtech.geowave.core.store.adapter.MockComponents.MockAbstractDataAdapter) Statistic(org.locationtech.geowave.core.store.api.Statistic) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) NumericRangeStatistic(org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic) MockComponents(org.locationtech.geowave.core.store.adapter.MockComponents) DataStore(org.locationtech.geowave.core.store.api.DataStore) GlobalVisibilityHandler(org.locationtech.geowave.core.store.data.visibility.GlobalVisibilityHandler) DataIdQuery(org.locationtech.geowave.core.store.query.constraints.DataIdQuery) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) StoreFactoryFamilySpi(org.locationtech.geowave.core.store.StoreFactoryFamilySpi) IndexImpl(org.locationtech.geowave.core.store.index.IndexImpl) GlobalVisibilityHandler(org.locationtech.geowave.core.store.data.visibility.GlobalVisibilityHandler) VisibilityHandler(org.locationtech.geowave.core.store.api.VisibilityHandler) Test(org.junit.Test)

Example 5 with CountStatistic

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic in project geowave by locationtech.

the class GeoWaveSpatialBinningStatisticsIT method testGeometry.

private static void testGeometry(final SimpleFeatureType featureType, final DataStore store) {
    final String geometryField = featureType.getGeometryDescriptor().getLocalName();
    final List<CountStatistic> stats = new ArrayList<>();
    for (final SpatialBinningType type : SpatialBinningType.values()) {
        for (final ComplexGeometryBinningOption complexGeometryOption : ComplexGeometryBinningOption.values()) {
            for (int precision = 1; precision < 4; precision++) {
                // S2 is more than twice as granular in its use of power of 2 "levels" as opposed to only
                // using the granularity of a character for geohash and H3
                // so double the precision for S2 to make it similar in scale
                final int finalPrecision = SpatialBinningType.S2.equals(type) ? precision * 2 : precision;
                final CountStatistic count = new CountStatistic(featureType.getTypeName());
                final SpatialFieldValueBinningStrategy strategy = new SpatialFieldValueBinningStrategy(geometryField);
                strategy.setComplexGeometry(complexGeometryOption);
                strategy.setPrecision(finalPrecision);
                strategy.setType(type);
                count.setTag(String.format("%s-%d-%s", type, finalPrecision, complexGeometryOption));
                count.setBinningStrategy(strategy);
                stats.add(count);
            }
        }
    }
    store.addStatistic(stats.toArray(new Statistic[stats.size()]));
    final CountStatistic referenceCountStat = new CountStatistic(featureType.getTypeName());
    store.addStatistic(referenceCountStat);
    final Long expectedCount = store.getStatisticValue(referenceCountStat);
    Assert.assertTrue("Must be at least one entry", expectedCount > 0);
    // sanity check scaling
    stats.stream().filter(s -> ((SpatialFieldValueBinningStrategy) s.getBinningStrategy()).getComplexGeometry().equals(ComplexGeometryBinningOption.USE_FULL_GEOMETRY_SCALE_BY_OVERLAP)).forEach(s -> Assert.assertEquals(String.format("%s failed scaled geometry", ((SpatialFieldValueBinningStrategy) s.getBinningStrategy()).getDefaultTag()), expectedCount, store.getStatisticValue(s), expectedCount * TYPE_TO_ERROR_THRESHOLD.get(((SpatialFieldValueBinningStrategy) s.getBinningStrategy()).getType())));
    // sanity check centroids
    stats.stream().filter(s -> ((SpatialFieldValueBinningStrategy) s.getBinningStrategy()).getComplexGeometry().equals(ComplexGeometryBinningOption.USE_CENTROID_ONLY)).forEach(s -> Assert.assertEquals(String.format("%s failed centroids at precision %d", ((SpatialFieldValueBinningStrategy) s.getBinningStrategy()).getType(), ((SpatialFieldValueBinningStrategy) s.getBinningStrategy()).getPrecision()), expectedCount, store.getStatisticValue(s)));
    // best way to sanity check full geometry is to perhaps check every bin count for centroid only
    // and for full geometry scale by overlap and make sure bin-by-bin every one of the full
    // geometry bins contains at least the count for either of the other 2 approaches (although
    // technically a centroid may be a bin that the full geometry doesn't even intersect so this is
    // not always a fair expectation but it'll suffice, particular when are precision only goes to 4
    // in this test
    final Map<BinningStrategyKey, Map<ByteArray, Long>> perBinResults = new HashMap<>();
    stats.stream().forEach(s -> {
        final Map<ByteArray, Long> results = new HashMap<>();
        ;
        perBinResults.put(new BinningStrategyKey((SpatialFieldValueBinningStrategy) s.getBinningStrategy()), results);
        try (CloseableIterator<Pair<ByteArray, Long>> it = store.getBinnedStatisticValues(s)) {
            while (it.hasNext()) {
                final Pair<ByteArray, Long> bin = it.next();
                Assert.assertFalse(results.containsKey(bin.getKey()));
                results.put(bin.getKey(), bin.getValue());
            }
        }
    });
    perBinResults.entrySet().stream().filter(e -> ComplexGeometryBinningOption.USE_FULL_GEOMETRY.equals(e.getKey().option)).forEach(entry -> {
        // get both the other complex binning options with matching type and precision and
        // make sure this full geometry count is at least the others for each bin
        final Map<ByteArray, Long> centroidResults = perBinResults.get(new BinningStrategyKey(entry.getKey().type, entry.getKey().precision, ComplexGeometryBinningOption.USE_CENTROID_ONLY));
        final Map<ByteArray, Long> scaledResults = perBinResults.get(new BinningStrategyKey(entry.getKey().type, entry.getKey().precision, ComplexGeometryBinningOption.USE_FULL_GEOMETRY_SCALE_BY_OVERLAP));
        entry.getValue().forEach((bin, count) -> {
            // make sure the scaled results exists for this bin, but is less than or equal to
            // this count
            final Long scaledResult = scaledResults.get(bin);
            Assert.assertNotNull(String.format("Scaled result doesn't exist for %s (%d) at bin %s", entry.getKey().type, entry.getKey().precision, entry.getKey().type.binToString(bin.getBytes())), scaledResult);
            Assert.assertTrue(String.format("Scaled result is greater than the full geometry for %s (%d) at bin %s", entry.getKey().type, entry.getKey().precision, entry.getKey().type.binToString(bin.getBytes())), scaledResult <= count);
            final Long centroidResult = centroidResults.get(bin);
            Assert.assertTrue(String.format("Centroid result is greater than the full geometry for %s (%d) at bin %s", entry.getKey().type, entry.getKey().precision, entry.getKey().type.binToString(bin.getBytes())), (centroidResult == null) || (centroidResult <= count));
        });
    });
}
Also used : FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Arrays(java.util.Arrays) AbstractFieldRetypingSource(org.locationtech.geowave.format.geotools.vector.AbstractFieldRetypingSource) LoggerFactory(org.slf4j.LoggerFactory) Coordinate(org.locationtech.jts.geom.Coordinate) NumericStatsStatistic(org.locationtech.geowave.core.store.statistics.field.NumericStatsStatistic) TestUtils(org.locationtech.geowave.test.TestUtils) FloatCompareUtils(org.locationtech.geowave.core.index.FloatCompareUtils) SimpleFeatureTypeBuilder(org.geotools.feature.simple.SimpleFeatureTypeBuilder) Pair(org.apache.commons.lang3.tuple.Pair) SimpleFeature(org.opengis.feature.simple.SimpleFeature) GeoToolsVectorDataOptions(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataOptions) AttributeDescriptor(org.opengis.feature.type.AttributeDescriptor) Map(java.util.Map) Statistic(org.locationtech.geowave.core.store.api.Statistic) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) AfterClass(org.junit.AfterClass) SpatialBinningType(org.locationtech.geowave.core.geotime.binning.SpatialBinningType) ImmutableMap(com.google.common.collect.ImmutableMap) Point(org.locationtech.jts.geom.Point) GeometryUtils(org.locationtech.geowave.core.geotime.util.GeometryUtils) DimensionalityType(org.locationtech.geowave.test.TestUtils.DimensionalityType) List(java.util.List) Entry(java.util.Map.Entry) GeoWaveITRunner(org.locationtech.geowave.test.GeoWaveITRunner) Geometry(org.locationtech.jts.geom.Geometry) ByteArray(org.locationtech.geowave.core.index.ByteArray) StatsAccumulator(org.locationtech.geowave.core.store.statistics.field.StatsAccumulator) BeforeClass(org.junit.BeforeClass) VectorQueryBuilder(org.locationtech.geowave.core.geotime.store.query.api.VectorQueryBuilder) BinConstraints(org.locationtech.geowave.core.store.api.BinConstraints) GeoWaveTestStore(org.locationtech.geowave.test.annotation.GeoWaveTestStore) Name(org.opengis.feature.type.Name) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) ArrayUtils(org.apache.commons.lang3.ArrayUtils) RetypingVectorDataPlugin(org.locationtech.geowave.format.geotools.vector.RetypingVectorDataPlugin) ComplexGeometryBinningOption(org.locationtech.geowave.core.geotime.binning.ComplexGeometryBinningOption) SimpleFeatureType(org.opengis.feature.simple.SimpleFeatureType) ArrayList(java.util.ArrayList) LocalFileIngestPlugin(org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin) Stats(org.locationtech.geowave.core.store.statistics.field.Stats) Logger(org.slf4j.Logger) MalformedURLException(java.net.MalformedURLException) Files(java.nio.file.Files) DataStore(org.locationtech.geowave.core.store.api.DataStore) IngestOptions(org.locationtech.geowave.core.store.api.IngestOptions) IOException(java.io.IOException) Test(org.junit.Test) File(java.io.File) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) TreeMap(java.util.TreeMap) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Paths(java.nio.file.Paths) SpatialFieldValueBinningStrategy(org.locationtech.geowave.core.geotime.store.statistics.binning.SpatialFieldValueBinningStrategy) Comparator(java.util.Comparator) Assert(org.junit.Assert) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) Envelope(org.locationtech.jts.geom.Envelope) SpatialBinningType(org.locationtech.geowave.core.geotime.binning.SpatialBinningType) HashMap(java.util.HashMap) SpatialFieldValueBinningStrategy(org.locationtech.geowave.core.geotime.store.statistics.binning.SpatialFieldValueBinningStrategy) ArrayList(java.util.ArrayList) Point(org.locationtech.jts.geom.Point) NumericStatsStatistic(org.locationtech.geowave.core.store.statistics.field.NumericStatsStatistic) Statistic(org.locationtech.geowave.core.store.api.Statistic) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) ByteArray(org.locationtech.geowave.core.index.ByteArray) ComplexGeometryBinningOption(org.locationtech.geowave.core.geotime.binning.ComplexGeometryBinningOption) CountStatistic(org.locationtech.geowave.core.store.statistics.adapter.CountStatistic) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) TreeMap(java.util.TreeMap) Pair(org.apache.commons.lang3.tuple.Pair)

Aggregations

CountStatistic (org.locationtech.geowave.core.store.statistics.adapter.CountStatistic)6 DataStore (org.locationtech.geowave.core.store.api.DataStore)5 SimpleFeature (org.opengis.feature.simple.SimpleFeature)4 Pair (org.apache.commons.lang3.tuple.Pair)3 Test (org.junit.Test)3 FeatureDataAdapter (org.locationtech.geowave.adapter.vector.FeatureDataAdapter)3 ByteArray (org.locationtech.geowave.core.index.ByteArray)3 Statistic (org.locationtech.geowave.core.store.api.Statistic)3 TimeRangeStatistic (org.locationtech.geowave.core.geotime.store.statistics.TimeRangeStatistic)2 SpatialFieldValueBinningStrategy (org.locationtech.geowave.core.geotime.store.statistics.binning.SpatialFieldValueBinningStrategy)2 NumericRangeStatistic (org.locationtech.geowave.core.store.statistics.field.NumericRangeStatistic)2 Coordinate (org.locationtech.jts.geom.Coordinate)2 ImmutableMap (com.google.common.collect.ImmutableMap)1 File (java.io.File)1 IOException (java.io.IOException)1 MalformedURLException (java.net.MalformedURLException)1 Files (java.nio.file.Files)1 Paths (java.nio.file.Paths)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1