Examples with CountValue org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue used on opensource projects

Example 1 with CountValue

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue in project geowave by locationtech.

the class StatServicesIT method testRecalcStats.

@Test
public void testRecalcStats() {
    final DataStore ds = dataStoreOptions.createDataStore();
    final CountValue expectedCount;
    final BoundingBoxValue expectedBoundingBox;
    try (CloseableIterator<CountValue> iter = ds.queryStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).build())) {
        assertTrue(iter.hasNext());
        expectedCount = iter.next();
        assertFalse(iter.hasNext());
    }
    try (CloseableIterator<BoundingBoxValue> iter = ds.queryStatistics(StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).build())) {
        assertTrue(iter.hasNext());
        expectedBoundingBox = iter.next();
        assertFalse(iter.hasNext());
    }
    TestUtils.assertStatusCode("Should successfully recalc stats for existent store", 200, statServiceClient.recalcStats(store_name));
    // Verify that the statistic values are still correct
    try (CloseableIterator<CountValue> iter = ds.queryStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).build())) {
        assertTrue(iter.hasNext());
        final CountValue newCount = iter.next();
        assertFalse(iter.hasNext());
        assertEquals(expectedCount.getValue(), newCount.getValue());
    }
    try (CloseableIterator<BoundingBoxValue> iter = ds.queryStatistics(StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).build())) {
        assertTrue(iter.hasNext());
        final BoundingBoxValue newBoundingBox = iter.next();
        assertFalse(iter.hasNext());
        assertEquals(expectedBoundingBox.getValue(), newBoundingBox.getValue());
    }
    TestUtils.assertStatusCode("Should successfully recalc stats for existent store and existent adapter", 200, statServiceClient.recalcStats(store_name, null, null, SimpleIngest.FEATURE_NAME, null, null, true, null));
    // Verify that the statistic values are still correct
    try (CloseableIterator<CountValue> iter = ds.queryStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).build())) {
        assertTrue(iter.hasNext());
        final CountValue newCount = iter.next();
        assertFalse(iter.hasNext());
        assertEquals(expectedCount.getValue(), newCount.getValue());
    }
    try (CloseableIterator<BoundingBoxValue> iter = ds.queryStatistics(StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).build())) {
        assertTrue(iter.hasNext());
        final BoundingBoxValue newBoundingBox = iter.next();
        assertFalse(iter.hasNext());
        assertEquals(expectedBoundingBox.getValue(), newBoundingBox.getValue());
    }
    // The following case should probably return a 404 based on the
    // situation described in the test description
    TestUtils.assertStatusCode("Returns a 400 status for recalc stats for existent store and nonexistent adapter", 400, statServiceClient.recalcStats(store_name, null, null, "nonexistent-adapter", null, null, true, null));
    muteLogging();
    TestUtils.assertStatusCode("Should fail to recalc stats for nonexistent store", 400, statServiceClient.recalcStats("nonexistent-store"));
    unmuteLogging();
}

Example 2 with CountValue

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue in project geowave by locationtech.

the class GeoWaveSparkIngestIT method testBasicSparkIngest.

@Test
public void testBasicSparkIngest() throws Exception {
    // ingest test points
    TestUtils.testSparkIngest(dataStore, DimensionalityType.SPATIAL, S3URL, GDELT_INPUT_FILES, "gdelt");
    final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        // query by the full bounding box, make sure there is more than
        // 0 count and make sure the count matches the number of results
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", new Integer(GDELT_COUNT), new Integer(resultCount));
    }
    // Clean up
    TestUtils.deleteAll(dataStore);
}

Example 3 with CountValue

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue in project geowave by locationtech.

the class BasicKafkaIT method testBasicIngestGpx.

@Test
public void testBasicIngestGpx() throws Exception {
    KafkaTestUtils.testKafkaStage(OSM_GPX_INPUT_DIR);
    KafkaTestUtils.testKafkaIngest(dataStorePluginOptions, false, OSM_GPX_INPUT_DIR);
    final DataStatisticsStore statsStore = dataStorePluginOptions.createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = dataStorePluginOptions.createAdapterStore();
    int adapterCount = 0;
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
        final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
        // then query it
        final GeometryFactory factory = new GeometryFactory();
        final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
        final Geometry spatialFilter = factory.toGeometry(env);
        final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
        final int resultCount = testQuery(adapter, query);
        assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
        assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
        assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", EXPECTED_COUNT_PER_ADAPTER_ID.get(adapter.getTypeName()), new Integer(resultCount));
        adapterCount++;
    }
    assertTrue("There should be exactly two adapters", (adapterCount == 2));
}

Example 4 with CountValue

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue in project geowave by locationtech.

the class GeoWaveVisibilityIT method testQuery.

public static void testQuery(final DataStore store, final DataStatisticsStore statsStore, final short internalAdapterId, final String[] auths, final boolean spatial, final int expectedResultCount, final int expectedNonNullFieldCount) {
    try (CloseableIterator<SimpleFeature> it = (CloseableIterator) store.query(QueryBuilder.newBuilder().setAuthorizations(auths).constraints(spatial ? new ExplicitSpatialQuery(new GeometryFactory().toGeometry(new Envelope(-1, 1, -1, 1))) : null).build())) {
        int resultCount = 0;
        int nonNullFieldsCount = 0;
        while (it.hasNext()) {
            final SimpleFeature feature = it.next();
            for (int a = 0; a < feature.getAttributeCount(); a++) {
                if (feature.getAttribute(a) != null) {
                    nonNullFieldsCount++;
                }
            }
            resultCount++;
        }
        Assert.assertEquals("Unexpected result count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedResultCount, resultCount);
        Assert.assertEquals("Unexpected non-null field count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedNonNullFieldCount, nonNullFieldsCount);
    }
    final Long count = (Long) store.aggregate(AggregationQueryBuilder.newBuilder().count(getType().getTypeName()).setAuthorizations(auths).constraints(spatial ? new ExplicitSpatialQuery(new GeometryFactory().toGeometry(new Envelope(-1, 1, -1, 1))) : null).build());
    Assert.assertEquals("Unexpected aggregation result count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedResultCount, count.intValue());
    final CountValue countStat = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(getType().getTypeName()).authorizations(auths).build());
    assertNotNull(countStat);
    Assert.assertEquals("Unexpected stats result count for " + (spatial ? "spatial query" : "full table scan") + " with auths " + Arrays.toString(auths), expectedResultCount, countStat.getValue().intValue());
}

Example 5 with CountValue

use of org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue in project geowave by locationtech.

the class GeoWaveVisibilityIT method testMixedVisibilityStatistics.

public static void testMixedVisibilityStatistics(final DataStorePluginOptions dataStoreOptions, final VisibilityHandler visibilityHandler, final int totalFeatures) {
    final SimpleFeatureBuilder bldr = new SimpleFeatureBuilder(getType());
    final FeatureDataAdapter adapter = new FeatureDataAdapter(getType());
    final DataStore store = dataStoreOptions.createDataStore();
    // Add some statistics
    final CountStatistic geomCount = new CountStatistic();
    geomCount.setTag("testGeom");
    geomCount.setTypeName(adapter.getTypeName());
    geomCount.setBinningStrategy(new FieldValueBinningStrategy("geometry"));
    final CountStatistic visCountC = new CountStatistic();
    visCountC.setTag("testC");
    visCountC.setTypeName(adapter.getTypeName());
    visCountC.setBinningStrategy(new FieldValueBinningStrategy("c"));
    final CountStatistic visCountAB = new CountStatistic();
    visCountAB.setTag("testAB");
    visCountAB.setTypeName(adapter.getTypeName());
    visCountAB.setBinningStrategy(new FieldValueBinningStrategy("a", "b"));
    // Specify visibility at the type level
    store.addType(adapter, visibilityHandler, Lists.newArrayList(geomCount, visCountC, visCountAB), TestUtils.DEFAULT_SPATIAL_INDEX);
    try (Writer<SimpleFeature> writer = store.createWriter(adapter.getTypeName())) {
        for (int i = 0; i < totalFeatures; i++) {
            bldr.set("a", A_FIELD_VALUES[i % 3]);
            bldr.set("b", B_FIELD_VALUES[i % 3]);
            bldr.set("c", C_FIELD_VALUES[i % 3]);
            bldr.set("geometry", new GeometryFactory().createPoint(new Coordinate(0, 0)));
            writer.write(bldr.buildFeature(Integer.toString(i)));
        }
    }
    // Since each field is only visible if you provide that field ID as an authorization, each
    // statistic should only reveal those counts if the appropriate authorization is set. Because
    // these statistics are using a field value binning strategy, the actual bins of the statistic
    // may reveal information that is not authorized to the user and should be hidden.
    final CountValue countCNoAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testC").build());
    assertEquals(0, countCNoAuth.getValue().longValue());
    // When providing the "c" auth, all values should be present
    final CountValue countCAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testC").addAuthorization("c").build());
    assertEquals(totalFeatures, countCAuth.getValue().longValue());
    // For the AB count statistic, the values should only be present if both "a" and "b"
    // authorizations are provided
    final CountValue countABNoAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").build());
    assertEquals(0, countABNoAuth.getValue().longValue());
    final CountValue countABOnlyAAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("a").build());
    assertEquals(0, countABOnlyAAuth.getValue().longValue());
    final CountValue countABOnlyBAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("b").build());
    assertEquals(0, countABOnlyBAuth.getValue().longValue());
    final CountValue countABAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("a").addAuthorization("b").build());
    assertEquals(totalFeatures, countABAuth.getValue().longValue());
    // It should also work if additional authorizations are provided
    final CountValue countABCAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testAB").addAuthorization("a").addAuthorization("b").addAuthorization("c").build());
    assertEquals(totalFeatures, countABCAuth.getValue().longValue());
    // Since the geometry field has no visibility, no authorizations should be required
    final CountValue countGeomNoAuth = store.aggregateStatistics(StatisticQueryBuilder.newBuilder(CountStatistic.STATS_TYPE).typeName(adapter.getTypeName()).tag("testGeom").build());
    assertEquals(totalFeatures, countGeomNoAuth.getValue().longValue());
}