use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.
the class DataStoreUtils method mergeData.
@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
try {
final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
mergingAdapters.put(adapter.getAdapterId(), adapter);
}
}
final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
final short[] adapterIds = new short[1];
for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
adapterIds[0] = adapter.getKey();
paramsBuilder.adapterIds(adapterIds);
try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
while (iterator.hasNext()) {
iterator.next();
}
} catch (final Exception e) {
LOGGER.error("Exception occurred while merging data.", e);
throw new RuntimeException(e);
}
}
} finally {
try {
deleter.close();
} catch (final Exception e) {
LOGGER.warn("Exception occurred when closing deleter.", e);
}
}
return true;
}
use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.
the class GeoWaveSparkIngestIT method testBasicSparkIngest.
@Test
public void testBasicSparkIngest() throws Exception {
// ingest test points
TestUtils.testSparkIngest(dataStore, DimensionalityType.SPATIAL, S3URL, GDELT_INPUT_FILES, "gdelt");
final DataStatisticsStore statsStore = dataStore.createDataStatisticsStore();
final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
// query by the full bounding box, make sure there is more than
// 0 count and make sure the count matches the number of results
final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
// then query it
final GeometryFactory factory = new GeometryFactory();
final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
final Geometry spatialFilter = factory.toGeometry(env);
final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
final int resultCount = testQuery(adapter, query);
assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", new Integer(GDELT_COUNT), new Integer(resultCount));
}
// Clean up
TestUtils.deleteAll(dataStore);
}
use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.
the class BasicKafkaIT method testBasicIngestGpx.
@Test
public void testBasicIngestGpx() throws Exception {
KafkaTestUtils.testKafkaStage(OSM_GPX_INPUT_DIR);
KafkaTestUtils.testKafkaIngest(dataStorePluginOptions, false, OSM_GPX_INPUT_DIR);
final DataStatisticsStore statsStore = dataStorePluginOptions.createDataStatisticsStore();
final PersistentAdapterStore adapterStore = dataStorePluginOptions.createAdapterStore();
int adapterCount = 0;
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
final BoundingBoxValue bboxValue = InternalStatisticsHelper.getFieldStatistic(statsStore, BoundingBoxStatistic.STATS_TYPE, adapter.getTypeName(), adapter.getFeatureType().getGeometryDescriptor().getLocalName());
final CountValue count = InternalStatisticsHelper.getDataTypeStatistic(statsStore, CountStatistic.STATS_TYPE, adapter.getTypeName());
// then query it
final GeometryFactory factory = new GeometryFactory();
final Envelope env = new Envelope(bboxValue.getMinX(), bboxValue.getMaxX(), bboxValue.getMinY(), bboxValue.getMaxY());
final Geometry spatialFilter = factory.toGeometry(env);
final QueryConstraints query = new ExplicitSpatialQuery(spatialFilter);
final int resultCount = testQuery(adapter, query);
assertTrue("'" + adapter.getTypeName() + "' adapter must have at least one element in its statistic", count.getValue() > 0);
assertEquals("'" + adapter.getTypeName() + "' adapter should have the same results from a spatial query of '" + env + "' as its total count statistic", count.getValue().intValue(), resultCount);
assertEquals("'" + adapter.getTypeName() + "' adapter entries ingested does not match expected count", EXPECTED_COUNT_PER_ADAPTER_ID.get(adapter.getTypeName()), new Integer(resultCount));
adapterCount++;
}
assertTrue("There should be exactly two adapters", (adapterCount == 2));
}
use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.
the class AbstractGeoWaveBasicVectorIT method testDeleteCQL.
protected void testDeleteCQL(final String cqlStr, final Index index) throws Exception {
LOGGER.info("bulk deleting using CQL: '" + cqlStr + "'");
final DataStore geowaveStore = getDataStorePluginOptions().createDataStore();
// Retrieve the feature adapter for the CQL query generator
final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
// Create the CQL query
final QueryConstraints query = OptimalCQLQuery.createOptimalQuery(cqlStr, (InternalGeotoolsFeatureDataAdapter<SimpleFeature>) adapter, null, null);
deleteInternal(geowaveStore, index, query);
}
}
use of org.locationtech.geowave.core.store.adapter.InternalDataAdapter in project geowave by locationtech.
the class AbstractGeoWaveBasicVectorIT method testQuery.
protected void testQuery(final URL savedFilterResource, final URL[] expectedResultsResources, final Pair<String, String> optimalCqlQueryGeometryAndTimeFields, final Index index, final String queryDescription, final CoordinateReferenceSystem crs, final boolean countDuplicates) throws IOException {
LOGGER.info("querying " + queryDescription);
final DataStore geowaveStore = getDataStorePluginOptions().createDataStore();
// this file is the filtered dataset (using the previous file as a
// filter) so use it to ensure the query worked
final QueryConstraints constraints = TestUtils.resourceToQuery(savedFilterResource, optimalCqlQueryGeometryAndTimeFields, true);
QueryBuilder<?, ?> bldr = QueryBuilder.newBuilder();
if (index != null) {
bldr = bldr.indexName(index.getName());
}
try (final CloseableIterator<?> actualResults = geowaveStore.query(bldr.constraints(constraints).build())) {
final ExpectedResults expectedResults = TestUtils.getExpectedResults(expectedResultsResources, crs);
int totalResults = 0;
final List<Long> actualCentroids = new ArrayList<>();
while (actualResults.hasNext()) {
final Object obj = actualResults.next();
if (obj instanceof SimpleFeature) {
final SimpleFeature result = (SimpleFeature) obj;
final long actualHashCentroid = TestUtils.hashCentroid((Geometry) result.getDefaultGeometry());
Assert.assertTrue("Actual result '" + result.toString() + "' not found in expected result set", expectedResults.hashedCentroids.contains(actualHashCentroid));
actualCentroids.add(actualHashCentroid);
totalResults++;
} else {
TestUtils.deleteAll(getDataStorePluginOptions());
Assert.fail("Actual result '" + obj.toString() + "' is not of type Simple Feature.");
}
}
for (final long l : actualCentroids) {
expectedResults.hashedCentroids.remove(l);
}
for (final long l : expectedResults.hashedCentroids) {
LOGGER.error("Missing expected hashed centroid: " + l);
}
if (expectedResults.count != totalResults) {
TestUtils.deleteAll(getDataStorePluginOptions());
}
Assert.assertEquals(expectedResults.count, totalResults);
final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
long statisticsResult = 0;
int duplicates = 0;
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
AggregationQueryBuilder<?, Long, ?, ?> aggBldr = AggregationQueryBuilder.newBuilder();
if (index != null) {
aggBldr = aggBldr.indexName(index.getName());
}
aggBldr = aggBldr.constraints(constraints);
if (countDuplicates) {
aggBldr.aggregate(internalDataAdapter.getTypeName(), (Aggregation) new DuplicateCountAggregation());
final DuplicateCount countResult = (DuplicateCount) geowaveStore.aggregate((AggregationQuery) aggBldr.build());
if (countResult != null) {
duplicates += countResult.count;
}
}
aggBldr.count(internalDataAdapter.getTypeName());
final Long countResult = geowaveStore.aggregate(aggBldr.build());
// results should already be aggregated, there should be
// exactly one value in this iterator
Assert.assertNotNull(countResult);
statisticsResult += countResult;
}
Assert.assertEquals(expectedResults.count, statisticsResult - duplicates);
}
}
Aggregations