use of org.locationtech.geowave.adapter.raster.adapter.merge.nodata.NoDataMergeStrategy in project geowave by locationtech.
the class KDERunner method run.
public void run() throws IOException {
initContext();
// Validate inputs
if (inputDataStore == null) {
LOGGER.error("You must supply an input datastore!");
throw new IOException("You must supply an input datastore!");
}
// Retrieve the feature adapters
final VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder();
List<String> featureTypeNames;
// If provided, just use the one
if (typeName != null) {
featureTypeNames = new ArrayList<>();
featureTypeNames.add(typeName);
} else {
// otherwise, grab all the feature adapters
featureTypeNames = FeatureDataUtils.getFeatureTypeNames(inputDataStore);
}
bldr.setTypeNames(featureTypeNames.toArray(new String[0]));
if (indexName != null) {
bldr.indexName(indexName);
}
Index inputPrimaryIndex = null;
final Index[] idxArray = inputDataStore.createDataStore().getIndices();
for (final Index idx : idxArray) {
if ((idx != null) && ((indexName == null) || indexName.equals(idx.getName()))) {
inputPrimaryIndex = idx;
break;
}
}
final CoordinateReferenceSystem inputIndexCrs = GeometryUtils.getIndexCrs(inputPrimaryIndex);
final String inputCrsCode = GeometryUtils.getCrsCode(inputIndexCrs);
Index outputPrimaryIndex = outputIndex;
CoordinateReferenceSystem outputIndexCrs = null;
final String outputCrsCode;
if (outputPrimaryIndex != null) {
outputIndexCrs = GeometryUtils.getIndexCrs(outputPrimaryIndex);
outputCrsCode = GeometryUtils.getCrsCode(outputIndexCrs);
} else {
final SpatialDimensionalityTypeProvider sdp = new SpatialDimensionalityTypeProvider();
final SpatialOptions so = sdp.createOptions();
so.setCrs(inputCrsCode);
outputPrimaryIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(so);
outputIndexCrs = inputIndexCrs;
outputCrsCode = inputCrsCode;
}
final CoordinateSystem cs = outputIndexCrs.getCoordinateSystem();
final CoordinateSystemAxis csx = cs.getAxis(0);
final CoordinateSystemAxis csy = cs.getAxis(1);
final double xMax = csx.getMaximumValue();
final double xMin = csx.getMinimumValue();
final double yMax = csy.getMaximumValue();
final double yMin = csy.getMinimumValue();
if ((xMax == Double.POSITIVE_INFINITY) || (xMin == Double.NEGATIVE_INFINITY) || (yMax == Double.POSITIVE_INFINITY) || (yMin == Double.NEGATIVE_INFINITY)) {
LOGGER.error("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
throw new RuntimeException("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
}
if (cqlFilter != null) {
bldr.constraints(bldr.constraintsFactory().cqlConstraints(cqlFilter));
}
// Load RDD from datastore
final RDDOptions kdeOpts = new RDDOptions();
kdeOpts.setMinSplits(minSplits);
kdeOpts.setMaxSplits(maxSplits);
kdeOpts.setQuery(bldr.build());
final Function<Double, Double> identity = x -> x;
final Function2<Double, Double, Double> sum = (final Double x, final Double y) -> {
return x + y;
};
final RasterDataAdapter adapter = RasterUtils.createDataAdapterTypeDouble(coverageName, KDEReducer.NUM_BANDS, tileSize, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, new NoDataMergeStrategy());
outputDataStore.createDataStore().addType(adapter, outputPrimaryIndex);
// The following "inner" variables are created to give access to member
// variables within lambda
// expressions
// tileSize;
final int innerTileSize = 1;
final String innerCoverageName = coverageName;
for (int level = minLevel; level <= maxLevel; level++) {
final int numXTiles = (int) Math.pow(2, level + 1);
final int numYTiles = (int) Math.pow(2, level);
// * tileSize;
final int numXPosts = numXTiles;
// * tileSize;
final int numYPosts = numYTiles;
final GeoWaveRDD kdeRDD = GeoWaveRDDLoader.loadRDD(session.sparkContext(), inputDataStore, kdeOpts);
JavaPairRDD<Double, Long> cells = kdeRDD.getRawRDD().flatMapToPair(new GeoWaveCellMapper(numXPosts, numYPosts, xMin, xMax, yMin, yMax, inputCrsCode, outputCrsCode)).combineByKey(identity, sum, sum).mapToPair(item -> item.swap());
cells = cells.partitionBy(new RangePartitioner(cells.getNumPartitions(), cells.rdd(), true, scala.math.Ordering.Double$.MODULE$, scala.reflect.ClassTag$.MODULE$.apply(Double.class))).sortByKey(false).cache();
final long count = cells.count();
if (count == 0) {
LOGGER.warn("No cells produced by KDE");
continue;
}
final double max = cells.first()._1;
JavaRDD<GridCoverage> rdd = cells.zipWithIndex().map(t -> {
final TileInfo tileInfo = fromCellIndexToTileInfo(t._1._2, numXPosts, numYPosts, numXTiles, numYTiles, xMin, xMax, yMin, yMax, innerTileSize);
final WritableRaster raster = RasterUtils.createRasterTypeDouble(NUM_BANDS, innerTileSize);
final double normalizedValue = t._1._1 / max;
// because we are using a Double as the key, the ordering
// isn't always completely reproducible as Double equals does not
// take into account an epsilon
final double percentile = (count - t._2) / ((double) count);
raster.setSample(tileInfo.x, tileInfo.y, 0, t._1._1);
raster.setSample(tileInfo.x, tileInfo.y, 1, normalizedValue);
raster.setSample(tileInfo.x, tileInfo.y, 2, percentile);
return RasterUtils.createCoverageTypeDouble(innerCoverageName, tileInfo.tileWestLon, tileInfo.tileEastLon, tileInfo.tileSouthLat, tileInfo.tileNorthLat, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, raster, GeometryUtils.DEFAULT_CRS_STR);
});
LOGGER.debug("Writing results to output store...");
if (tileSize > 1) {
// byte[] adapterBytes = PersistenceUtils.toBinary(adapter);
// byte[] indexBytes = PersistenceUtils.toBinary(outputPrimaryIndex);
rdd = rdd.flatMapToPair(new TransformTileSize(adapter, outputPrimaryIndex)).groupByKey().map(new MergeOverlappingTiles(adapter, outputPrimaryIndex));
}
RDDUtils.writeRasterToGeoWave(jsc.sc(), outputPrimaryIndex, outputDataStore, adapter, rdd);
LOGGER.debug("Results successfully written!");
}
}
use of org.locationtech.geowave.adapter.raster.adapter.merge.nodata.NoDataMergeStrategy in project geowave by locationtech.
the class RasterIngestRunner method processPreviousScene.
@SuppressWarnings({ "rawtypes", "unchecked" })
protected void processPreviousScene() {
if (!ingestOptions.isCoveragePerBand()) {
// ingest as single image for all bands
if (!lastSceneBands.isEmpty()) {
// we are sorting by band name to ensure a consistent order for
// bands
final TreeMap<String, RasterBandData> sceneData = new TreeMap<>();
Writer writer;
// get coverage info, ensuring that all coverage names are the
// same
String coverageName = null;
for (final SimpleFeature band : lastSceneBands) {
RasterBandData bandData;
try {
bandData = getBandData(band);
if (coverageName == null) {
coverageName = bandData.name;
} else if (!coverageName.equals(bandData.name)) {
LOGGER.warn("Unable to use band data as the band coverage name '" + bandData.name + "' is unexpectedly different from default name '" + coverageName + "'");
}
final String bandName = band.getAttribute(BandFeatureIterator.BAND_ATTRIBUTE_NAME).toString();
sceneData.put(bandName, bandData);
} catch (IOException | TemplateException e) {
LOGGER.warn("Unable to read band data", e);
}
}
if (coverageName == null) {
LOGGER.warn("No valid bands found for scene");
lastSceneBands.clear();
return;
}
final GridCoverage2D mergedCoverage;
if (sceneData.size() == 1) {
mergedCoverage = sceneData.firstEntry().getValue().coverage;
} else {
final CoverageProcessor processor = CoverageProcessor.getInstance();
final AbstractOperation op = (AbstractOperation) processor.getOperation("BandMerge");
final ParameterValueGroup params = op.getParameters();
final List<GridCoverage2D> sources = new ArrayList<>();
for (final RasterBandData bandData : sceneData.values()) {
sources.add(bandData.coverage);
}
params.parameter("Sources").setValue(sources);
params.parameter(BandMerge.TRANSFORM_CHOICE).setValue(TransformList.FIRST.toString());
mergedCoverage = (GridCoverage2D) op.doOperation(params, null);
}
final String[] thisSceneBands = sceneData.keySet().toArray(new String[] {});
if (bandsIngested == null) {
// this means this is the first scene
// setup adapter and other required info
final Map<String, String> metadata = new HashMap<>();
final double[][] noDataValues = new double[sceneData.size()][];
int b = 0;
// merge metadata from all readers
for (final RasterBandData bandData : sceneData.values()) {
try {
final String[] metadataNames = bandData.reader.getMetadataNames();
if ((metadataNames != null) && (metadataNames.length > 0)) {
for (final String metadataName : metadataNames) {
metadata.put(metadataName, bandData.reader.getMetadataValue(metadataName));
}
}
} catch (final Exception e) {
LOGGER.warn("Unable to get metadata for coverage '" + coverageName + "'.", e);
}
noDataValues[b++] = new double[] { bandData.nodataValue };
}
final RasterDataAdapter adapter = new RasterDataAdapter(coverageName, metadata, mergedCoverage, ingestOptions.getTileSize(), ingestOptions.isCreatePyramid(), ingestOptions.isCreateHistogram(), noDataValues, new NoDataMergeStrategy());
store.addType(adapter, indices);
writer = store.createWriter(adapter.getTypeName());
writerCache.put(coverageName, writer);
bandsIngested = thisSceneBands;
} else if (!Arrays.equals(bandsIngested, thisSceneBands)) {
LOGGER.warn("The bands in this scene ('" + Arrays.toString(thisSceneBands) + "') differ from the previous scene ('" + Arrays.toString(bandsIngested) + "'). To merge bands all scenes must use the same bands. Skipping scene'" + lastSceneBands.get(0).getAttribute(SceneFeatureIterator.ENTITY_ID_ATTRIBUTE_NAME) + "'.");
lastSceneBands.clear();
return;
} else {
writer = writerCache.get(coverageName);
if (writer == null) {
LOGGER.warn("Unable to find writer for coverage '" + coverageName + "'. Skipping scene'" + lastSceneBands.get(0).getAttribute(SceneFeatureIterator.ENTITY_ID_ATTRIBUTE_NAME) + "'.");
lastSceneBands.clear();
return;
}
}
writer.write(mergedCoverage);
lastSceneBands.clear();
}
}
}
use of org.locationtech.geowave.adapter.raster.adapter.merge.nodata.NoDataMergeStrategy in project geowave by locationtech.
the class GeoWaveVisibilityIT method ingestAndQueryComplexVisibilityRasters.
@SuppressWarnings({ "unchecked", "rawtypes" })
private void ingestAndQueryComplexVisibilityRasters(final String coverageName, final int tileSize, final double westLon, final double eastLon, final double southLat, final double northLat) throws IOException {
// Create two test rasters
final int numBands = 8;
final DataStore dataStore = dataStoreOptions.createDataStore();
final RasterDataAdapter adapter = RasterUtils.createDataAdapterTypeDouble(coverageName, numBands, tileSize, new NoDataMergeStrategy());
final WritableRaster raster1 = RasterUtils.createRasterTypeDouble(numBands, tileSize);
final WritableRaster raster2 = RasterUtils.createRasterTypeDouble(numBands, tileSize);
TestUtils.fillTestRasters(raster1, raster2, tileSize);
dataStore.addType(adapter, TestUtils.DEFAULT_SPATIAL_INDEX);
try (Writer writer = dataStore.createWriter(adapter.getTypeName())) {
// Write the first raster w/ vis info
writer.write(RasterUtils.createCoverageTypeDouble(coverageName, westLon, eastLon, southLat, northLat, raster1), getRasterVisWriter("(a&b)|c"));
// Write the second raster w/ no vis info
writer.write(RasterUtils.createCoverageTypeDouble(coverageName, westLon, eastLon, southLat, northLat, raster2));
}
try (CloseableIterator<?> it = dataStore.query(QueryBuilder.newBuilder().addTypeName(coverageName).build())) {
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 0; b < numBands; b++) {
final double p0 = raster.getSampleDouble(x, y, b);
final double p1 = raster2.getSampleDouble(x, y, b);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=" + b, p0, p1, 0.0);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
// just the second raster back
try (CloseableIterator<?> it = dataStore.query(QueryBuilder.newBuilder().addTypeName(coverageName).addAuthorization("a").build())) {
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 0; b < numBands; b++) {
final double p0 = raster.getSampleDouble(x, y, b);
final double p1 = raster2.getSampleDouble(x, y, b);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=" + b, p0, p1, 0.0);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
// just the second raster back
try (CloseableIterator<?> it = dataStore.query(QueryBuilder.newBuilder().addTypeName(coverageName).addAuthorization("b").build())) {
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 0; b < numBands; b++) {
final double p0 = raster.getSampleDouble(x, y, b);
final double p1 = raster2.getSampleDouble(x, y, b);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=" + b, p0, p1, 0.0);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
try (CloseableIterator<?> it = dataStore.query(QueryBuilder.newBuilder().addTypeName(coverageName).addAuthorization("c").build())) {
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
// 7 has the upper quadrant as NaN and the rest set
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 1; b < 7; b++) {
final double pExp = TestUtils.getTileValue(x, y, b, tileSize);
final double pAct = raster.getSampleDouble(x, y, b);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=" + b, pExp, pAct, 0.0);
}
if ((y % 2) == 0) {
final double pExp = TestUtils.getTileValue(x, y, 0, tileSize);
final double pAct = raster.getSampleDouble(x, y, 0);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=0", pExp, pAct, 0.0);
} else {
final double pAct = raster.getSampleDouble(x, y, 0);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=0", Double.NaN, pAct, 0.0);
}
if ((x > ((tileSize * 3) / 4)) && (y > ((tileSize * 3) / 4))) {
final double pAct = raster.getSampleDouble(x, y, 7);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=7", Double.NaN, pAct, 0.0);
} else {
final double pExp = TestUtils.getTileValue(x, y, 7, tileSize);
final double pAct = raster.getSampleDouble(x, y, 7);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=7", pExp, pAct, 0.0);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
try (CloseableIterator<?> it = dataStore.query(QueryBuilder.newBuilder().addTypeName(coverageName).addAuthorization("a").addAuthorization("b").build())) {
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
// 7 has the upper quadrant as NaN and the rest set
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 1; b < 7; b++) {
final double pExp = TestUtils.getTileValue(x, y, b, tileSize);
final double pAct = raster.getSampleDouble(x, y, b);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=" + b, pExp, pAct, 0.0);
}
if ((y % 2) == 0) {
final double pExp = TestUtils.getTileValue(x, y, 0, tileSize);
final double pAct = raster.getSampleDouble(x, y, 0);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=0", pExp, pAct, 0.0);
} else {
final double pAct = raster.getSampleDouble(x, y, 0);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=0", Double.NaN, pAct, 0.0);
}
if ((x > ((tileSize * 3) / 4)) && (y > ((tileSize * 3) / 4))) {
final double pAct = raster.getSampleDouble(x, y, 7);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=7", Double.NaN, pAct, 0.0);
} else {
final double pExp = TestUtils.getTileValue(x, y, 7, tileSize);
final double pAct = raster.getSampleDouble(x, y, 7);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=7", pExp, pAct, 0.0);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
}
use of org.locationtech.geowave.adapter.raster.adapter.merge.nodata.NoDataMergeStrategy in project geowave by locationtech.
the class GeoWaveVisibilityIT method ingestAndQueryMixedVisibilityRasters.
private void ingestAndQueryMixedVisibilityRasters(final String coverageName, final int tileSize, final double westLon, final double eastLon, final double southLat, final double northLat) throws IOException {
// Create two test rasters
final int numBands = 8;
final DataStore dataStore = dataStoreOptions.createDataStore();
final RasterDataAdapter adapter = RasterUtils.createDataAdapterTypeDouble(coverageName, numBands, tileSize, new NoDataMergeStrategy());
final WritableRaster raster1 = RasterUtils.createRasterTypeDouble(numBands, tileSize);
final WritableRaster raster2 = RasterUtils.createRasterTypeDouble(numBands, tileSize);
TestUtils.fillTestRasters(raster1, raster2, tileSize);
dataStore.addType(adapter, TestUtils.DEFAULT_SPATIAL_INDEX);
try (Writer writer = dataStore.createWriter(adapter.getTypeName())) {
// Write the first raster w/ vis info
writer.write(RasterUtils.createCoverageTypeDouble(coverageName, westLon, eastLon, southLat, northLat, raster1), getRasterVisWriter("a"));
// Write the second raster w/ no vis info
writer.write(RasterUtils.createCoverageTypeDouble(coverageName, westLon, eastLon, southLat, northLat, raster2));
}
try (CloseableIterator<?> it = dataStore.query(QueryBuilder.newBuilder().addTypeName(coverageName).build())) {
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 0; b < numBands; b++) {
final double p0 = raster.getSampleDouble(x, y, b);
final double p1 = raster2.getSampleDouble(x, y, b);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=" + b, p0, p1, 0.0);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
try (CloseableIterator<?> it = dataStore.query(QueryBuilder.newBuilder().addTypeName(coverageName).addAuthorization("a").build())) {
final GridCoverage coverage = (GridCoverage) it.next();
final Raster raster = coverage.getRenderedImage().getData();
Assert.assertEquals(tileSize, raster.getWidth());
Assert.assertEquals(tileSize, raster.getHeight());
// 7 has the upper quadrant as NaN and the rest set
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 1; b < 7; b++) {
final double pExp = TestUtils.getTileValue(x, y, b, tileSize);
final double pAct = raster.getSampleDouble(x, y, b);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=" + b, pExp, pAct, 0.0);
}
if ((y % 2) == 0) {
final double pExp = TestUtils.getTileValue(x, y, 0, tileSize);
final double pAct = raster.getSampleDouble(x, y, 0);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=0", pExp, pAct, 0.0);
} else {
final double pAct = raster.getSampleDouble(x, y, 0);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=0", Double.NaN, pAct, 0.0);
}
if ((x > ((tileSize * 3) / 4)) && (y > ((tileSize * 3) / 4))) {
final double pAct = raster.getSampleDouble(x, y, 7);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=7", Double.NaN, pAct, 0.0);
} else {
final double pExp = TestUtils.getTileValue(x, y, 7, tileSize);
final double pAct = raster.getSampleDouble(x, y, 7);
Assert.assertEquals("x=" + x + ",y=" + y + ",b=7", pExp, pAct, 0.0);
}
}
}
// there should be exactly one
Assert.assertFalse(it.hasNext());
}
}
use of org.locationtech.geowave.adapter.raster.adapter.merge.nodata.NoDataMergeStrategy in project geowave by locationtech.
the class GeoWaveBasicCustomCRSRasterIT method ingestGeneralPurpose.
private void ingestGeneralPurpose(final String coverageName, final int tileSize, final double westLon, final double eastLon, final double southLat, final double northLat, final int numBands, final int numRasters, final RasterTileMergeStrategy<?> mergeStrategy) throws IOException {
// just ingest a number of rasters
final DataStore dataStore = dataStoreOptions.createDataStore();
final RasterDataAdapter basicAdapter = RasterUtils.createDataAdapterTypeDouble(coverageName, numBands, tileSize, new NoDataMergeStrategy());
final RasterDataAdapter mergeStrategyOverriddenAdapter = new RasterDataAdapter(basicAdapter, coverageName, mergeStrategy);
basicAdapter.getMetadata().put("test-key", "test-value");
dataStore.addType(mergeStrategyOverriddenAdapter, TestUtils.createWebMercatorSpatialIndex());
try (Writer writer = dataStore.createWriter(mergeStrategyOverriddenAdapter.getTypeName())) {
for (int r = 0; r < numRasters; r++) {
final WritableRaster raster = RasterUtils.createRasterTypeDouble(numBands, tileSize);
for (int x = 0; x < tileSize; x++) {
for (int y = 0; y < tileSize; y++) {
for (int b = 0; b < numBands; b++) {
raster.setSample(x, y, b, TestUtils.getTileValue(x, y, b, r, tileSize));
}
}
}
writer.write(createCoverageTypeDouble(coverageName, westLon, eastLon, southLat, northLat, raster));
}
}
}
Aggregations