Search in sources :

Example 1 with TransientAdapterStore

use of org.locationtech.geowave.core.store.adapter.TransientAdapterStore in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}
Also used : FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Arrays(java.util.Arrays) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) URL(java.net.URL) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) CommonIndexAggregation(org.locationtech.geowave.core.store.query.aggregate.CommonIndexAggregation) LoggerFactory(org.slf4j.LoggerFactory) Aggregation(org.locationtech.geowave.core.store.api.Aggregation) MathUtils(org.apache.commons.math.util.MathUtils) TestUtils(org.locationtech.geowave.test.TestUtils) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) ByteBuffer(java.nio.ByteBuffer) TimeDescriptors(org.locationtech.geowave.core.geotime.util.TimeDescriptors) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore) StatisticValue(org.locationtech.geowave.core.store.api.StatisticValue) Pair(org.apache.commons.lang3.tuple.Pair) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Map(java.util.Map) Statistic(org.locationtech.geowave.core.store.api.Statistic) Maps(jersey.repackaged.com.google.common.collect.Maps) Persistable(org.locationtech.geowave.core.index.persist.Persistable) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FieldStatistic(org.locationtech.geowave.core.store.api.FieldStatistic) StatisticQuery(org.locationtech.geowave.core.store.api.StatisticQuery) Set(java.util.Set) ManualOperationParams(org.locationtech.geowave.core.cli.parser.ManualOperationParams) DimensionalityType(org.locationtech.geowave.test.TestUtils.DimensionalityType) ExpectedResults(org.locationtech.geowave.test.TestUtils.ExpectedResults) ConfigOptions(org.locationtech.geowave.core.cli.operations.config.options.ConfigOptions) List(java.util.List) VectorLocalExportOptions(org.locationtech.geowave.adapter.vector.export.VectorLocalExportOptions) Entry(java.util.Map.Entry) DataIdQuery(org.locationtech.geowave.core.store.query.constraints.DataIdQuery) Geometry(org.locationtech.jts.geom.Geometry) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) CoordinateReferenceSystem(org.opengis.referencing.crs.CoordinateReferenceSystem) ByteArray(org.locationtech.geowave.core.index.ByteArray) AggregationQuery(org.locationtech.geowave.core.store.api.AggregationQuery) BeforeClass(org.junit.BeforeClass) AddStoreCommand(org.locationtech.geowave.core.store.cli.store.AddStoreCommand) AggregationQueryBuilder(org.locationtech.geowave.core.store.api.AggregationQueryBuilder) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) CommonIndexedPersistenceEncoding(org.locationtech.geowave.core.store.data.CommonIndexedPersistenceEncoding) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) LocalFileIngestPlugin(org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin) Calendar(java.util.Calendar) Lists(com.google.common.collect.Lists) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) StatisticQueryBuilder(org.locationtech.geowave.core.store.api.StatisticQueryBuilder) QueryBuilder(org.locationtech.geowave.core.store.api.QueryBuilder) Index(org.locationtech.geowave.core.store.api.Index) StatisticsIngestCallback(org.locationtech.geowave.core.store.statistics.StatisticsIngestCallback) BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) GeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.GeotoolsFeatureDataAdapter) Logger(org.slf4j.Logger) DataStore(org.locationtech.geowave.core.store.api.DataStore) OptimalCQLQuery(org.locationtech.geowave.core.geotime.store.query.OptimalCQLQuery) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) IngestCallback(org.locationtech.geowave.core.store.callback.IngestCallback) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) VectorLocalExportCommand(org.locationtech.geowave.adapter.vector.export.VectorLocalExportCommand) File(java.io.File) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) InternalGeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.InternalGeotoolsFeatureDataAdapter) MemoryAdapterStore(org.locationtech.geowave.core.store.memory.MemoryAdapterStore) Filter(org.opengis.filter.Filter) ZipUtils(org.locationtech.geowave.adapter.raster.util.ZipUtils) Assert(org.junit.Assert) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) Envelope(org.locationtech.jts.geom.Envelope) Assert.assertEquals(org.junit.Assert.assertEquals) StatisticValue(org.locationtech.geowave.core.store.api.StatisticValue) HashMap(java.util.HashMap) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) Envelope(org.locationtech.jts.geom.Envelope) URL(java.net.URL) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) Entry(java.util.Map.Entry) Statistic(org.locationtech.geowave.core.store.api.Statistic) FieldStatistic(org.locationtech.geowave.core.store.api.FieldStatistic) BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) ByteArray(org.locationtech.geowave.core.index.ByteArray) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore) MemoryAdapterStore(org.locationtech.geowave.core.store.memory.MemoryAdapterStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) GeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.GeotoolsFeatureDataAdapter) InternalGeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.InternalGeotoolsFeatureDataAdapter) Map(java.util.Map) HashMap(java.util.HashMap)

Example 2 with TransientAdapterStore

use of org.locationtech.geowave.core.store.adapter.TransientAdapterStore in project geowave by locationtech.

the class GeoWaveOutputFormat method getRecordWriter.

@Override
public RecordWriter<GeoWaveOutputKey<Object>, Object> getRecordWriter(final TaskAttemptContext context) throws IOException, InterruptedException {
    try {
        final Map<String, String> configOptions = getStoreOptionsMap(context);
        final IndexStore persistentIndexStore = GeoWaveStoreFinder.createIndexStore(configOptions);
        final DataStore dataStore = GeoWaveStoreFinder.createDataStore(configOptions);
        final Index[] indices = JobContextIndexStore.getIndices(context);
        if (LOGGER.isDebugEnabled()) {
            final StringBuilder sbDebug = new StringBuilder();
            sbDebug.append("Config Options: ");
            for (final Map.Entry<String, String> entry : configOptions.entrySet()) {
                sbDebug.append(entry.getKey() + "/" + entry.getValue() + ", ");
            }
            sbDebug.append("\n\tIndices Size: " + indices.length);
            sbDebug.append("\n\tpersistentIndexStore: " + persistentIndexStore);
            final String filename = "/META-INF/services/org.locationtech.geowave.core.store.StoreFactoryFamilySpi";
            final InputStream is = context.getClass().getResourceAsStream(filename);
            if (is == null) {
                sbDebug.append("\n\tStoreFactoryFamilySpi: Unable to open file '" + filename + "'");
            } else {
                sbDebug.append("\n\tStoreFactoryFamilySpi: " + IOUtils.toString(is, "UTF-8"));
                is.close();
            }
            LOGGER.debug(sbDebug.toString());
        }
        for (final Index i : indices) {
            if (!persistentIndexStore.indexExists(i.getName())) {
                dataStore.addIndex(i);
            }
        }
        final TransientAdapterStore jobContextAdapterStore = GeoWaveConfiguratorBase.getJobContextAdapterStore(CLASS, context);
        final IndexStore jobContextIndexStore = new JobContextIndexStore(context, persistentIndexStore);
        return new GeoWaveRecordWriter(dataStore, jobContextIndexStore, jobContextAdapterStore);
    } catch (final Exception e) {
        throw new IOException(e);
    }
}
Also used : JobContextIndexStore(org.locationtech.geowave.mapreduce.JobContextIndexStore) InputStream(java.io.InputStream) Index(org.locationtech.geowave.core.store.api.Index) IOException(java.io.IOException) IOException(java.io.IOException) DataStore(org.locationtech.geowave.core.store.api.DataStore) MapReduceDataStore(org.locationtech.geowave.mapreduce.MapReduceDataStore) HashMap(java.util.HashMap) Map(java.util.Map) JobContextIndexStore(org.locationtech.geowave.mapreduce.JobContextIndexStore) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore)

Aggregations

IOException (java.io.IOException)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 Lists (com.google.common.collect.Lists)1 File (java.io.File)1 InputStream (java.io.InputStream)1 URISyntaxException (java.net.URISyntaxException)1 URL (java.net.URL)1 ByteBuffer (java.nio.ByteBuffer)1 SimpleDateFormat (java.text.SimpleDateFormat)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Calendar (java.util.Calendar)1 Date (java.util.Date)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Entry (java.util.Map.Entry)1 Set (java.util.Set)1 Maps (jersey.repackaged.com.google.common.collect.Maps)1 FileUtils (org.apache.commons.io.FileUtils)1