Examples with NumericHistogramStatistic org.locationtech.geowave.core.store.statistics.field.NumericHistogramStatistic used on opensource projects

Example 1 with NumericHistogramStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericHistogramStatistic in project geowave by locationtech.

the class GeoWaveStatisticsIT method testAddStatistic.

@Test
public void testAddStatistic() {
    final DataStore ds = dataStore.createDataStore();
    final NumericRangeStatistic longitudeRange = new NumericRangeStatistic(SimpleIngest.FEATURE_NAME, "Longitude");
    final NumericRangeStatistic latitudeRange = new NumericRangeStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    final TimeRangeStatistic timeRange = new TimeRangeStatistic(SimpleIngest.FEATURE_NAME, "TimeStamp");
    final NumericStatsStatistic latitudeStats = new NumericStatsStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    final BloomFilterStatistic latitudeBloomFilter = new BloomFilterStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    final NumericHistogramStatistic latitudeHistogram = new NumericHistogramStatistic(SimpleIngest.FEATURE_NAME, "Latitude");
    ds.addStatistic(longitudeRange, timeRange, latitudeStats, latitudeBloomFilter, latitudeHistogram);
    ds.addEmptyStatistic(latitudeRange);
    try (CloseableIterator<NumericRangeValue> iterator = ds.queryStatistics(StatisticQueryBuilder.newBuilder(NumericRangeStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).fieldName("Longitude").build())) {
        assertTrue(iterator.hasNext());
        final NumericRangeValue value = iterator.next();
        assertEquals(-165.0, value.getMin(), 0.1);
        assertEquals(180.0, value.getMax(), 0.1);
        assertFalse(iterator.hasNext());
    }
    try (CloseableIterator<NumericRangeValue> iterator = ds.queryStatistics(StatisticQueryBuilder.newBuilder(NumericRangeStatistic.STATS_TYPE).typeName(SimpleIngest.FEATURE_NAME).fieldName("Latitude").build())) {
        // We only calculated stats for Longitude
        assertTrue(iterator.hasNext());
        assertFalse(iterator.next().isSet());
        assertFalse(iterator.hasNext());
    }
    final Interval interval = ds.getStatisticValue(timeRange);
    try (CloseableIterator<SimpleFeature> it = ds.query(VectorQueryBuilder.newBuilder().build())) {
        long min = Long.MAX_VALUE, max = Long.MIN_VALUE;
        while (it.hasNext()) {
            final long time = ((Date) it.next().getAttribute("TimeStamp")).getTime();
            min = Math.min(min, time);
            max = Math.max(max, time);
        }
        assertEquals(min, interval.getStart().toEpochMilli());
        assertEquals(max, interval.getEnd().toEpochMilli());
    }
    final Stats stats = ds.getStatisticValue(latitudeStats);
    assertEquals(20L, stats.count());
    assertEquals(-90.0, stats.min(), 0.1);
    assertEquals(85.0, stats.max(), 0.1);
    assertEquals(-0.5, stats.mean(), 0.1);
    assertEquals(53.47, stats.populationStandardDeviation(), 0.1);
    final BloomFilter<CharSequence> bloomFilter = ds.getStatisticValue(latitudeBloomFilter);
    boolean expectLat = true;
    for (double lat = -90; lat <= 90; lat += 5) {
        if (expectLat) {
            assertTrue(bloomFilter.mightContain(Double.toString(lat)));
        } else {
            assertFalse(bloomFilter.mightContain(Double.toString(lat)));
        }
        // and forth each iteration, 3 times it stays true at these latitudes
        if (!DoubleMath.fuzzyEquals(-40, lat, 0.1) && !DoubleMath.fuzzyEquals(25, lat, 0.1) && !DoubleMath.fuzzyEquals(80, lat, 0.1)) {
            expectLat = !expectLat;
        }
    }
    final NumericHistogram histogram = ds.getStatisticValue(latitudeHistogram);
    assertEquals(20L, histogram.getTotalCount(), 0.1);
    assertEquals(-90.0, histogram.getMinValue(), 0.1);
    assertEquals(85.0, histogram.getMaxValue(), 0.1);
    assertEquals(0.0, histogram.quantile(0.5), 0.1);
}

Example 2 with NumericHistogramStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericHistogramStatistic in project geowave by locationtech.

the class NumericHistogramStatisticsTest method testRapidIncreaseInRange.

@Test
public void testRapidIncreaseInRange() {
    final NumericHistogramStatistic stat = new NumericHistogramStatistic("", "pop");
    final NumericHistogramValue statValue = stat.createEmpty();
    final Random rand = new Random(7777);
    double next = 1;
    for (int i = 0; i < 100; i++) {
        next = next + (rand.nextDouble() * 100.0);
        statValue.entryIngested(dataAdapter, create(next));
    }
    for (long i = 0; i < 100; i++) {
        final NumericHistogramValue statValue2 = stat.createEmpty();
        for (int j = 0; j < 100; j++) {
            statValue2.entryIngested(dataAdapter, create(4839000434.547854578 * rand.nextDouble() * rand.nextGaussian()));
        }
        byte[] b = statValue2.toBinary();
        statValue2.fromBinary(b);
        b = statValue.toBinary();
        statValue.fromBinary(b);
        statValue.merge(statValue2);
    }
}

Example 3 with NumericHistogramStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericHistogramStatistic in project geowave by locationtech.

the class NumericHistogramStatisticsTest method testNegative.

@Test
public void testNegative() {
    final NumericHistogramStatistic stat = new NumericHistogramStatistic("", "pop");
    final NumericHistogramValue statValue = stat.createEmpty();
    final Random rand = new Random(7777);
    statValue.entryIngested(dataAdapter, create(-100.0));
    statValue.entryIngested(dataAdapter, create(-101.0));
    statValue.entryIngested(dataAdapter, create(-2.0));
    double next = -1;
    for (int i = 0; i < 10000; i++) {
        next = next - (Math.round(rand.nextDouble()));
        statValue.entryIngested(dataAdapter, create(next));
    }
    final NumericHistogramValue statValue2 = stat.createEmpty();
    final double start2 = next;
    double min = 0;
    for (long i = 0; i < 10000; i++) {
        final double val = next - (long) (1000 * rand.nextDouble());
        statValue2.entryIngested(dataAdapter, create(val));
        min = Math.min(val, min);
    }
    final double skewvalue = next - (1000 * rand.nextDouble());
    final SimpleFeature skewedFeature = create(skewvalue);
    for (int i = 0; i < 10000; i++) {
        statValue2.entryIngested(dataAdapter, skewedFeature);
    }
    assertEquals(1.0, statValue2.cdf(0), 0.00001);
    final byte[] b = statValue2.toBinary();
    statValue2.fromBinary(b);
    assertEquals(0.0, statValue2.cdf(min), 0.00001);
    statValue.merge(statValue2);
    assertEquals(1.0, statValue.cdf(0), 0.00001);
    assertEquals(0.66, statValue.cdf(start2), 0.01);
    assertEquals(30003, sum(statValue.count(10)));
    final double r = statValue.percentPopulationOverRange(skewvalue - 1, skewvalue + 1);
    assertTrue((r > 0.3) && (r < 0.35));
}

Example 4 with NumericHistogramStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericHistogramStatistic in project geowave by locationtech.

the class NumericHistogramStatisticsTest method testPositive.

@Test
public void testPositive() {
    final NumericHistogramStatistic stat = new NumericHistogramStatistic("", "pop");
    final NumericHistogramValue statValue = stat.createEmpty();
    final Random rand = new Random(7777);
    statValue.entryIngested(dataAdapter, create(100.0));
    statValue.entryIngested(dataAdapter, create(101.0));
    statValue.entryIngested(dataAdapter, create(2.0));
    double next = 1;
    for (int i = 0; i < 10000; i++) {
        next = next + (Math.round(rand.nextDouble()));
        statValue.entryIngested(dataAdapter, create(next));
    }
    final NumericHistogramValue statValue2 = stat.createEmpty();
    final double start2 = next;
    double max = 0;
    for (long i = 0; i < 10000; i++) {
        final double val = next + (1000 * rand.nextDouble());
        statValue2.entryIngested(dataAdapter, create(val));
        max = Math.max(val, max);
    }
    final double skewvalue = next + (1000 * rand.nextDouble());
    final SimpleFeature skewedFeature = create(skewvalue);
    for (int i = 0; i < 10000; i++) {
        statValue2.entryIngested(dataAdapter, skewedFeature);
    // skewedFeature.setAttribute("pop", Long.valueOf(next + (long)
    // (1000 * rand.nextDouble())));
    }
    final byte[] b = statValue2.toBinary();
    statValue2.fromBinary(b);
    assertEquals(1.0, statValue2.cdf(max + 1), 0.00001);
    statValue.merge(statValue2);
    assertEquals(1.0, statValue.cdf(max + 1), 0.00001);
    assertEquals(0.33, statValue.cdf(start2), 0.01);
    assertEquals(30003, sum(statValue.count(10)));
    final double r = statValue.percentPopulationOverRange(skewvalue - 1, skewvalue + 1);
    assertTrue((r > 0.3) && (r < 0.35));
}

Example 5 with NumericHistogramStatistic

use of org.locationtech.geowave.core.store.statistics.field.NumericHistogramStatistic in project geowave by locationtech.

the class NumericHistogramStatisticsTest method testMix.

@Test
public void testMix() {
    final NumericHistogramStatistic stat = new NumericHistogramStatistic("", "pop");
    final NumericHistogramValue statValue = stat.createEmpty();
    final Random rand = new Random(7777);
    double min = 0;
    double max = 0;
    double next = 0;
    for (int i = 1; i < 300; i++) {
        final NumericHistogramValue statValue2 = stat.createEmpty();
        final double m = 10000.0 * Math.pow(10.0, ((i / 100) + 1));
        if (i == 50) {
            next = 0.0;
        } else if (i == 100) {
            next = Double.NaN;
        } else if (i == 150) {
            next = Double.MAX_VALUE;
        } else if (i == 200) {
            next = Integer.MAX_VALUE;
        } else if (i == 225) {
            next = Integer.MIN_VALUE;
        } else {
            next = (m * rand.nextDouble() * MathUtils.sign(rand.nextGaussian()));
        }
        statValue2.entryIngested(dataAdapter, create(next));
        if (!Double.isNaN(next)) {
            max = Math.max(next, max);
            min = Math.min(next, min);
            stat.fromBinary(stat.toBinary());
            statValue2.fromBinary(statValue2.toBinary());
            statValue.merge(statValue2);
        }
    }
    assertEquals(0.5, statValue.cdf(0), 0.1);
    assertEquals(0.0, statValue.cdf(min), 0.00001);
    assertEquals(1.0, statValue.cdf(max), 0.00001);
    assertEquals(298, sum(statValue.count(10)));
}