Examples with MemoryAdapterStore org.locationtech.geowave.core.store.memory.MemoryAdapterStore used on opensource projects

Example 1 with MemoryAdapterStore

use of org.locationtech.geowave.core.store.memory.MemoryAdapterStore in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}

Example 2 with MemoryAdapterStore

use of org.locationtech.geowave.core.store.memory.MemoryAdapterStore in project geowave by locationtech.

the class SerializableAdapterStoreTest method testSerialization.

@Test
public void testSerialization() throws ClassNotFoundException, IOException {
    final SimpleFeatureType ftype = AnalyticFeature.createGeometryFeatureAdapter("centroid", new String[] { "extra1" }, BasicFeatureTypes.DEFAULT_NAMESPACE, ClusteringUtils.CLUSTERING_CRS).getFeatureType();
    final FeatureDataAdapter adapter = new FeatureDataAdapter(ftype);
    final SerializableAdapterStore store = new SerializableAdapterStore(new MemoryAdapterStore(new DataTypeAdapter<?>[] { adapter }));
    final String id = "centroid";
    assertNotNull(checkSerialization(store).getAdapter(id));
}

Example 3 with MemoryAdapterStore

use of org.locationtech.geowave.core.store.memory.MemoryAdapterStore in project geowave by locationtech.

the class SplitsProviderIT method getSplitsMSE.

private double getSplitsMSE(final QueryConstraints query, final int minSplits, final int maxSplits) {
    // get splits and create reader for each RangeLocationPair, then summing
    // up the rows for each split
    List<InputSplit> splits = null;
    final MapReduceDataStore dataStore = (MapReduceDataStore) dataStorePluginOptions.createDataStore();
    final PersistentAdapterStore as = dataStorePluginOptions.createAdapterStore();
    final InternalAdapterStore ias = dataStorePluginOptions.createInternalAdapterStore();
    final MapReduceDataStoreOperations ops = (MapReduceDataStoreOperations) dataStorePluginOptions.createDataStoreOperations();
    final IndexStore is = dataStorePluginOptions.createIndexStore();
    final AdapterIndexMappingStore aim = dataStorePluginOptions.createAdapterIndexMappingStore();
    final DataStatisticsStore stats = dataStorePluginOptions.createDataStatisticsStore();
    final MemoryAdapterStore mas = new MemoryAdapterStore();
    mas.addAdapter(fda);
    try {
        splits = dataStore.getSplits(new CommonQueryOptions(), new FilterByTypeQueryOptions<>(new String[] { fda.getTypeName() }), new QuerySingleIndex(idx.getName()), new EverythingQuery(), mas, aim, stats, ias, is, new JobContextImpl(new Configuration(), new JobID()), minSplits, maxSplits);
    } catch (final IOException e) {
        LOGGER.error("IOException thrown when calling getSplits", e);
    } catch (final InterruptedException e) {
        LOGGER.error("InterruptedException thrown when calling getSplits", e);
    }
    final double[] observed = new double[splits.size()];
    int totalCount = 0;
    int currentSplit = 0;
    for (final InputSplit split : splits) {
        int countPerSplit = 0;
        if (GeoWaveInputSplit.class.isAssignableFrom(split.getClass())) {
            final GeoWaveInputSplit gwSplit = (GeoWaveInputSplit) split;
            for (final String indexName : gwSplit.getIndexNames()) {
                final SplitInfo splitInfo = gwSplit.getInfo(indexName);
                for (final RangeLocationPair p : splitInfo.getRangeLocationPairs()) {
                    final RecordReaderParams readerParams = new RecordReaderParams(splitInfo.getIndex(), as, aim, ias, new short[] { ias.getAdapterId(fda.getTypeName()) }, null, null, null, splitInfo.isMixedVisibility(), splitInfo.isAuthorizationsLimiting(), splitInfo.isClientsideRowMerging(), p.getRange(), null, null);
                    try (RowReader<?> reader = ops.createReader(readerParams)) {
                        while (reader.hasNext()) {
                            reader.next();
                            countPerSplit++;
                        }
                    } catch (final Exception e) {
                        LOGGER.error("Exception thrown when calling createReader", e);
                    }
                }
            }
        }
        totalCount += countPerSplit;
        observed[currentSplit] = countPerSplit;
        currentSplit++;
    }
    final double expected = 1.0 / splits.size();
    double sum = 0;
    for (int i = 0; i < observed.length; i++) {
        sum += Math.pow((observed[i] / totalCount) - expected, 2);
    }
    return sum / splits.size();
}