use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.
the class AbstractGeoWaveBasicVectorIT method testStats.
@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
// In the multithreaded case, only test min/max and count. Stats will be
// ingested/ in a different order and will not match.
final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
final Map<String, StatisticsCache> statsCache = new HashMap<>();
final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
for (final URL inputFile : inputFiles) {
LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
while (dataIterator.hasNext()) {
final GeoWaveData<SimpleFeature> data = dataIterator.next();
final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
// it should be a statistical data adapter
if (adapter instanceof DefaultStatisticsProvider) {
StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
if (cachedValues == null) {
cachedValues = new StatisticsCache(adapter, crs);
statsCache.put(adapter.getTypeName(), cachedValues);
}
cachedValues.entryIngested(data.getValue());
}
}
}
}
final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
Assert.assertNotNull(cachedValue);
final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
int statsCount = 0;
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
while (statsIterator.hasNext()) {
statsIterator.next();
statsCount++;
}
}
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
while (statsIterator.hasNext()) {
statsIterator.next();
statsCount++;
}
}
Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
StatisticValue<Object> actual;
if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
} else {
actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
}
assertEquals(expectedValues.getValue().getValue(), actual.getValue());
}
}
// finally check the one stat that is more manually calculated -
// the bounding box
StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
validateBBox(bboxStat.getValue(), cachedValue);
// now make sure it works without giving field name because there is only one geometry field
// anyways
query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
validateBBox(bboxStat.getValue(), cachedValue);
final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
}
}
use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.
the class IngestFromKafkaDriver method processMessage.
protected synchronized <T> void processMessage(final T dataRecord, final KafkaIngestRunData ingestRunData, final GeoWaveAvroFormatPlugin<T, ?> plugin) throws IOException {
final IngestPluginBase<T, ?> ingestPlugin = plugin.getIngestWithAvroPlugin();
final IndexProvider indexProvider = plugin;
final Map<String, Writer> writerMap = new HashMap<>();
final Map<String, Index> indexMap = new HashMap<>();
for (final Index index : indices) {
indexMap.put(index.getName(), index);
}
final Index[] requiredIndices = indexProvider.getRequiredIndices();
if ((requiredIndices != null) && (requiredIndices.length > 0)) {
for (final Index requiredIndex : requiredIndices) {
indexMap.put(requiredIndex.getName(), requiredIndex);
}
}
try (CloseableIterator<?> geowaveDataIt = ingestPlugin.toGeoWaveData(dataRecord, indexMap.keySet().toArray(new String[0]))) {
while (geowaveDataIt.hasNext()) {
final GeoWaveData<?> geowaveData = (GeoWaveData<?>) geowaveDataIt.next();
final DataTypeAdapter adapter = ingestRunData.getDataAdapter(geowaveData);
if (adapter == null) {
LOGGER.warn("Adapter not found for " + geowaveData.getValue());
continue;
}
Writer indexWriter = writerMap.get(adapter.getTypeName());
if (indexWriter == null) {
final List<Index> indexList = new ArrayList<>();
for (final String indexName : geowaveData.getIndexNames()) {
final Index index = indexMap.get(indexName);
if (index == null) {
LOGGER.warn("Index '" + indexName + "' not found for " + geowaveData.getValue());
continue;
}
indexList.add(index);
}
indexWriter = ingestRunData.getIndexWriter(adapter, visibilityHandler, indexList.toArray(new Index[indexList.size()]));
writerMap.put(adapter.getTypeName(), indexWriter);
}
indexWriter.write(geowaveData.getValue());
}
}
}
use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.
the class IngestMapper method map.
@Override
protected void map(final AvroKey key, final NullWritable value, final org.apache.hadoop.mapreduce.Mapper.Context context) throws IOException, InterruptedException {
try (CloseableIterator<GeoWaveData> data = ingestWithMapper.toGeoWaveData(key.datum(), indexNames)) {
while (data.hasNext()) {
final GeoWaveData d = data.next();
context.write(new GeoWaveOutputKey<>(d), d.getValue());
}
}
}
use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.
the class GeoLifeIngestPlugin method toGeoWaveDataInternal.
@Override
protected CloseableIterator<GeoWaveData<SimpleFeature>> toGeoWaveDataInternal(final AvroWholeFile hfile, final String[] indexNames) {
final List<GeoWaveData<SimpleFeature>> featureData = new ArrayList<>();
final InputStream in = new ByteArrayInputStream(hfile.getOriginalFile().array());
final InputStreamReader isr = new InputStreamReader(in, StringUtils.getGeoWaveCharset());
final BufferedReader br = new BufferedReader(isr);
int pointInstance = 0;
final List<Coordinate> pts = new ArrayList<>();
final String trackId = FilenameUtils.getName(hfile.getOriginalFilePath().toString());
String line;
Date startTimeStamp = null;
Date endTimeStamp = null;
String timestring = "";
final GeometryFactory geometryFactory = new GeometryFactory();
double currLat;
double currLng;
try {
while ((line = br.readLine()) != null) {
final String[] vals = line.split(",");
if (vals.length != 7) {
continue;
}
currLat = GeometryUtils.adjustCoordinateDimensionToRange(Double.parseDouble(vals[0]), crs, 1);
currLng = GeometryUtils.adjustCoordinateDimensionToRange(Double.parseDouble(vals[1]), crs, 0);
final Coordinate cord = new Coordinate(currLng, currLat);
pts.add(cord);
geolifePointBuilder.set("geometry", geometryFactory.createPoint(cord));
geolifePointBuilder.set("trackid", trackId);
geolifePointBuilder.set("pointinstance", pointInstance);
pointInstance++;
timestring = vals[5] + " " + vals[6];
final Date ts = GeoLifeUtils.parseDate(timestring);
geolifePointBuilder.set("Timestamp", ts);
if (startTimeStamp == null) {
startTimeStamp = ts;
}
endTimeStamp = ts;
geolifePointBuilder.set("Latitude", currLat);
geolifePointBuilder.set("Longitude", currLng);
Double elevation = Double.parseDouble(vals[3]);
if (elevation == -777) {
elevation = null;
}
geolifePointBuilder.set("Elevation", elevation);
featureData.add(new GeoWaveData<>(pointKey, indexNames, geolifePointBuilder.buildFeature(trackId + "_" + pointInstance)));
}
geolifeTrackBuilder.set("geometry", geometryFactory.createLineString(pts.toArray(new Coordinate[pts.size()])));
geolifeTrackBuilder.set("StartTimeStamp", startTimeStamp);
geolifeTrackBuilder.set("EndTimeStamp", endTimeStamp);
if ((endTimeStamp != null) && (startTimeStamp != null)) {
geolifeTrackBuilder.set("Duration", endTimeStamp.getTime() - startTimeStamp.getTime());
}
geolifeTrackBuilder.set("NumberPoints", pointInstance);
geolifeTrackBuilder.set("TrackId", trackId);
featureData.add(new GeoWaveData<>(trackKey, indexNames, geolifeTrackBuilder.buildFeature(trackId)));
} catch (final IOException e) {
LOGGER.warn("Error reading line from file: " + hfile.getOriginalFilePath(), e);
} catch (final ParseException e) {
LOGGER.error("Error parsing time string: " + timestring, e);
} finally {
IOUtils.closeQuietly(br);
IOUtils.closeQuietly(isr);
IOUtils.closeQuietly(in);
}
return new CloseableIterator.Wrapper<>(featureData.iterator());
}
use of org.locationtech.geowave.core.store.ingest.GeoWaveData in project geowave by locationtech.
the class GEOLIFEIngestTest method testIngest.
@Test
public void testIngest() throws IOException {
final URL toIngest = this.getClass().getClassLoader().getResource(filePath);
assertTrue(GeoLifeUtils.validate(toIngest));
final CloseableIterator<GeoWaveData<SimpleFeature>> features = ingester.toGeoWaveData(toIngest, new String[] { "123" });
assertTrue((features != null) && features.hasNext());
int featureCount = 0;
while (features.hasNext()) {
final GeoWaveData<SimpleFeature> feature = features.next();
if (isValidGeoLifeFeature(feature)) {
featureCount++;
}
}
features.close();
final boolean readExpectedCount = (featureCount == expectedCount);
if (!readExpectedCount) {
System.out.println("Expected " + expectedCount + " features, ingested " + featureCount);
}
assertTrue(readExpectedCount);
}
Aggregations