Search in sources :

Example 1 with GeoWaveData

use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}
Also used : FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) Arrays(java.util.Arrays) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) URL(java.net.URL) Date(java.util.Date) URISyntaxException(java.net.URISyntaxException) CommonIndexAggregation(org.locationtech.geowave.core.store.query.aggregate.CommonIndexAggregation) LoggerFactory(org.slf4j.LoggerFactory) Aggregation(org.locationtech.geowave.core.store.api.Aggregation) MathUtils(org.apache.commons.math.util.MathUtils) TestUtils(org.locationtech.geowave.test.TestUtils) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) ByteBuffer(java.nio.ByteBuffer) TimeDescriptors(org.locationtech.geowave.core.geotime.util.TimeDescriptors) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore) StatisticValue(org.locationtech.geowave.core.store.api.StatisticValue) Pair(org.apache.commons.lang3.tuple.Pair) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Map(java.util.Map) Statistic(org.locationtech.geowave.core.store.api.Statistic) Maps(jersey.repackaged.com.google.common.collect.Maps) Persistable(org.locationtech.geowave.core.index.persist.Persistable) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FieldStatistic(org.locationtech.geowave.core.store.api.FieldStatistic) StatisticQuery(org.locationtech.geowave.core.store.api.StatisticQuery) Set(java.util.Set) ManualOperationParams(org.locationtech.geowave.core.cli.parser.ManualOperationParams) DimensionalityType(org.locationtech.geowave.test.TestUtils.DimensionalityType) ExpectedResults(org.locationtech.geowave.test.TestUtils.ExpectedResults) ConfigOptions(org.locationtech.geowave.core.cli.operations.config.options.ConfigOptions) List(java.util.List) VectorLocalExportOptions(org.locationtech.geowave.adapter.vector.export.VectorLocalExportOptions) Entry(java.util.Map.Entry) DataIdQuery(org.locationtech.geowave.core.store.query.constraints.DataIdQuery) Geometry(org.locationtech.jts.geom.Geometry) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) CoordinateReferenceSystem(org.opengis.referencing.crs.CoordinateReferenceSystem) ByteArray(org.locationtech.geowave.core.index.ByteArray) AggregationQuery(org.locationtech.geowave.core.store.api.AggregationQuery) BeforeClass(org.junit.BeforeClass) AddStoreCommand(org.locationtech.geowave.core.store.cli.store.AddStoreCommand) AggregationQueryBuilder(org.locationtech.geowave.core.store.api.AggregationQueryBuilder) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) CommonIndexedPersistenceEncoding(org.locationtech.geowave.core.store.data.CommonIndexedPersistenceEncoding) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) LocalFileIngestPlugin(org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin) Calendar(java.util.Calendar) Lists(com.google.common.collect.Lists) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) StatisticQueryBuilder(org.locationtech.geowave.core.store.api.StatisticQueryBuilder) QueryBuilder(org.locationtech.geowave.core.store.api.QueryBuilder) Index(org.locationtech.geowave.core.store.api.Index) StatisticsIngestCallback(org.locationtech.geowave.core.store.statistics.StatisticsIngestCallback) BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) GeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.GeotoolsFeatureDataAdapter) Logger(org.slf4j.Logger) DataStore(org.locationtech.geowave.core.store.api.DataStore) OptimalCQLQuery(org.locationtech.geowave.core.geotime.store.query.OptimalCQLQuery) IOException(java.io.IOException) FileUtils(org.apache.commons.io.FileUtils) IngestCallback(org.locationtech.geowave.core.store.callback.IngestCallback) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) VectorLocalExportCommand(org.locationtech.geowave.adapter.vector.export.VectorLocalExportCommand) File(java.io.File) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) InternalGeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.InternalGeotoolsFeatureDataAdapter) MemoryAdapterStore(org.locationtech.geowave.core.store.memory.MemoryAdapterStore) Filter(org.opengis.filter.Filter) ZipUtils(org.locationtech.geowave.adapter.raster.util.ZipUtils) Assert(org.junit.Assert) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) Envelope(org.locationtech.jts.geom.Envelope) Assert.assertEquals(org.junit.Assert.assertEquals) StatisticValue(org.locationtech.geowave.core.store.api.StatisticValue) HashMap(java.util.HashMap) GeoToolsVectorDataStoreIngestPlugin(org.locationtech.geowave.format.geotools.vector.GeoToolsVectorDataStoreIngestPlugin) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) BoundingBoxValue(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic.BoundingBoxValue) Envelope(org.locationtech.jts.geom.Envelope) URL(java.net.URL) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) Entry(java.util.Map.Entry) Statistic(org.locationtech.geowave.core.store.api.Statistic) FieldStatistic(org.locationtech.geowave.core.store.api.FieldStatistic) BoundingBoxStatistic(org.locationtech.geowave.core.geotime.store.statistics.BoundingBoxStatistic) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) ByteArray(org.locationtech.geowave.core.index.ByteArray) TransientAdapterStore(org.locationtech.geowave.core.store.adapter.TransientAdapterStore) MemoryAdapterStore(org.locationtech.geowave.core.store.memory.MemoryAdapterStore) SimpleFeature(org.opengis.feature.simple.SimpleFeature) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) FeatureDataAdapter(org.locationtech.geowave.adapter.vector.FeatureDataAdapter) GeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.GeotoolsFeatureDataAdapter) InternalGeotoolsFeatureDataAdapter(org.locationtech.geowave.core.geotime.store.InternalGeotoolsFeatureDataAdapter) Map(java.util.Map) HashMap(java.util.HashMap)

Example 2 with GeoWaveData

use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.

the class IngestFromKafkaDriver method processMessage.

protected synchronized <T> void processMessage(final T dataRecord, final KafkaIngestRunData ingestRunData, final GeoWaveAvroFormatPlugin<T, ?> plugin) throws IOException {
    final IngestPluginBase<T, ?> ingestPlugin = plugin.getIngestWithAvroPlugin();
    final IndexProvider indexProvider = plugin;
    final Map<String, Writer> writerMap = new HashMap<>();
    final Map<String, Index> indexMap = new HashMap<>();
    for (final Index index : indices) {
        indexMap.put(index.getName(), index);
    }
    final Index[] requiredIndices = indexProvider.getRequiredIndices();
    if ((requiredIndices != null) && (requiredIndices.length > 0)) {
        for (final Index requiredIndex : requiredIndices) {
            indexMap.put(requiredIndex.getName(), requiredIndex);
        }
    }
    try (CloseableIterator<?> geowaveDataIt = ingestPlugin.toGeoWaveData(dataRecord, indexMap.keySet().toArray(new String[0]))) {
        while (geowaveDataIt.hasNext()) {
            final GeoWaveData<?> geowaveData = (GeoWaveData<?>) geowaveDataIt.next();
            final DataTypeAdapter adapter = ingestRunData.getDataAdapter(geowaveData);
            if (adapter == null) {
                LOGGER.warn("Adapter not found for " + geowaveData.getValue());
                continue;
            }
            Writer indexWriter = writerMap.get(adapter.getTypeName());
            if (indexWriter == null) {
                final List<Index> indexList = new ArrayList<>();
                for (final String indexName : geowaveData.getIndexNames()) {
                    final Index index = indexMap.get(indexName);
                    if (index == null) {
                        LOGGER.warn("Index '" + indexName + "' not found for " + geowaveData.getValue());
                        continue;
                    }
                    indexList.add(index);
                }
                indexWriter = ingestRunData.getIndexWriter(adapter, visibilityHandler, indexList.toArray(new Index[indexList.size()]));
                writerMap.put(adapter.getTypeName(), indexWriter);
            }
            indexWriter.write(geowaveData.getValue());
        }
    }
}
Also used : HashMap(java.util.HashMap) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) ArrayList(java.util.ArrayList) Index(org.locationtech.geowave.core.store.api.Index) IndexProvider(org.locationtech.geowave.core.store.ingest.IndexProvider) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) Writer(org.locationtech.geowave.core.store.api.Writer)

Example 3 with GeoWaveData

use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.

the class IngestMapper method map.

@Override
protected void map(final AvroKey key, final NullWritable value, final org.apache.hadoop.mapreduce.Mapper.Context context) throws IOException, InterruptedException {
    try (CloseableIterator<GeoWaveData> data = ingestWithMapper.toGeoWaveData(key.datum(), indexNames)) {
        while (data.hasNext()) {
            final GeoWaveData d = data.next();
            context.write(new GeoWaveOutputKey<>(d), d.getValue());
        }
    }
}
Also used : GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData)

Example 4 with GeoWaveData

use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.

the class GeoLifeIngestPlugin method toGeoWaveDataInternal.

@Override
protected CloseableIterator<GeoWaveData<SimpleFeature>> toGeoWaveDataInternal(final AvroWholeFile hfile, final String[] indexNames) {
    final List<GeoWaveData<SimpleFeature>> featureData = new ArrayList<>();
    final InputStream in = new ByteArrayInputStream(hfile.getOriginalFile().array());
    final InputStreamReader isr = new InputStreamReader(in, StringUtils.getGeoWaveCharset());
    final BufferedReader br = new BufferedReader(isr);
    int pointInstance = 0;
    final List<Coordinate> pts = new ArrayList<>();
    final String trackId = FilenameUtils.getName(hfile.getOriginalFilePath().toString());
    String line;
    Date startTimeStamp = null;
    Date endTimeStamp = null;
    String timestring = "";
    final GeometryFactory geometryFactory = new GeometryFactory();
    double currLat;
    double currLng;
    try {
        while ((line = br.readLine()) != null) {
            final String[] vals = line.split(",");
            if (vals.length != 7) {
                continue;
            }
            currLat = GeometryUtils.adjustCoordinateDimensionToRange(Double.parseDouble(vals[0]), crs, 1);
            currLng = GeometryUtils.adjustCoordinateDimensionToRange(Double.parseDouble(vals[1]), crs, 0);
            final Coordinate cord = new Coordinate(currLng, currLat);
            pts.add(cord);
            geolifePointBuilder.set("geometry", geometryFactory.createPoint(cord));
            geolifePointBuilder.set("trackid", trackId);
            geolifePointBuilder.set("pointinstance", pointInstance);
            pointInstance++;
            timestring = vals[5] + " " + vals[6];
            final Date ts = GeoLifeUtils.parseDate(timestring);
            geolifePointBuilder.set("Timestamp", ts);
            if (startTimeStamp == null) {
                startTimeStamp = ts;
            }
            endTimeStamp = ts;
            geolifePointBuilder.set("Latitude", currLat);
            geolifePointBuilder.set("Longitude", currLng);
            Double elevation = Double.parseDouble(vals[3]);
            if (elevation == -777) {
                elevation = null;
            }
            geolifePointBuilder.set("Elevation", elevation);
            featureData.add(new GeoWaveData<>(pointKey, indexNames, geolifePointBuilder.buildFeature(trackId + "_" + pointInstance)));
        }
        geolifeTrackBuilder.set("geometry", geometryFactory.createLineString(pts.toArray(new Coordinate[pts.size()])));
        geolifeTrackBuilder.set("StartTimeStamp", startTimeStamp);
        geolifeTrackBuilder.set("EndTimeStamp", endTimeStamp);
        if ((endTimeStamp != null) && (startTimeStamp != null)) {
            geolifeTrackBuilder.set("Duration", endTimeStamp.getTime() - startTimeStamp.getTime());
        }
        geolifeTrackBuilder.set("NumberPoints", pointInstance);
        geolifeTrackBuilder.set("TrackId", trackId);
        featureData.add(new GeoWaveData<>(trackKey, indexNames, geolifeTrackBuilder.buildFeature(trackId)));
    } catch (final IOException e) {
        LOGGER.warn("Error reading line from file: " + hfile.getOriginalFilePath(), e);
    } catch (final ParseException e) {
        LOGGER.error("Error parsing time string: " + timestring, e);
    } finally {
        IOUtils.closeQuietly(br);
        IOUtils.closeQuietly(isr);
        IOUtils.closeQuietly(in);
    }
    return new CloseableIterator.Wrapper<>(featureData.iterator());
}
Also used : GeometryFactory(org.locationtech.jts.geom.GeometryFactory) InputStreamReader(java.io.InputStreamReader) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) ArrayList(java.util.ArrayList) IOException(java.io.IOException) Date(java.util.Date) GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) ByteArrayInputStream(java.io.ByteArrayInputStream) Coordinate(org.locationtech.jts.geom.Coordinate) BufferedReader(java.io.BufferedReader) ParseException(java.text.ParseException)

Example 5 with GeoWaveData

use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.

the class GEOLIFEIngestTest method testIngest.

@Test
public void testIngest() throws IOException {
    final URL toIngest = this.getClass().getClassLoader().getResource(filePath);
    assertTrue(GeoLifeUtils.validate(toIngest));
    final CloseableIterator<GeoWaveData<SimpleFeature>> features = ingester.toGeoWaveData(toIngest, new String[] { "123" });
    assertTrue((features != null) && features.hasNext());
    int featureCount = 0;
    while (features.hasNext()) {
        final GeoWaveData<SimpleFeature> feature = features.next();
        if (isValidGeoLifeFeature(feature)) {
            featureCount++;
        }
    }
    features.close();
    final boolean readExpectedCount = (featureCount == expectedCount);
    if (!readExpectedCount) {
        System.out.println("Expected " + expectedCount + " features, ingested " + featureCount);
    }
    assertTrue(readExpectedCount);
}
Also used : GeoWaveData(org.locationtech.geowave.core.store.ingest.GeoWaveData) URL(java.net.URL) SimpleFeature(org.opengis.feature.simple.SimpleFeature) Test(org.junit.Test)

Aggregations

GeoWaveData (org.locationtech.geowave.core.store.ingest.GeoWaveData)16 SimpleFeature (org.opengis.feature.simple.SimpleFeature)9 ArrayList (java.util.ArrayList)8 IOException (java.io.IOException)6 URL (java.net.URL)6 Test (org.junit.Test)6 Date (java.util.Date)5 Coordinate (org.locationtech.jts.geom.Coordinate)4 BufferedReader (java.io.BufferedReader)3 ByteArrayInputStream (java.io.ByteArrayInputStream)3 InputStream (java.io.InputStream)3 InputStreamReader (java.io.InputStreamReader)3 HashMap (java.util.HashMap)3 GeometryFactory (org.locationtech.jts.geom.GeometryFactory)3 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)2 File (java.io.File)2 DataTypeAdapter (org.locationtech.geowave.core.store.api.DataTypeAdapter)2 Index (org.locationtech.geowave.core.store.api.Index)2 CoordinateReferenceSystem (org.opengis.referencing.crs.CoordinateReferenceSystem)2 Lists (com.google.common.collect.Lists)1