use of org.locationtech.geowave.core.geotime.index.SpatialDimensionalityTypeProvider in project geowave by locationtech.
the class KDERunner method run.
public void run() throws IOException {
initContext();
// Validate inputs
if (inputDataStore == null) {
LOGGER.error("You must supply an input datastore!");
throw new IOException("You must supply an input datastore!");
}
// Retrieve the feature adapters
final VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder();
List<String> featureTypeNames;
// If provided, just use the one
if (typeName != null) {
featureTypeNames = new ArrayList<>();
featureTypeNames.add(typeName);
} else {
// otherwise, grab all the feature adapters
featureTypeNames = FeatureDataUtils.getFeatureTypeNames(inputDataStore);
}
bldr.setTypeNames(featureTypeNames.toArray(new String[0]));
if (indexName != null) {
bldr.indexName(indexName);
}
Index inputPrimaryIndex = null;
final Index[] idxArray = inputDataStore.createDataStore().getIndices();
for (final Index idx : idxArray) {
if ((idx != null) && ((indexName == null) || indexName.equals(idx.getName()))) {
inputPrimaryIndex = idx;
break;
}
}
final CoordinateReferenceSystem inputIndexCrs = GeometryUtils.getIndexCrs(inputPrimaryIndex);
final String inputCrsCode = GeometryUtils.getCrsCode(inputIndexCrs);
Index outputPrimaryIndex = outputIndex;
CoordinateReferenceSystem outputIndexCrs = null;
final String outputCrsCode;
if (outputPrimaryIndex != null) {
outputIndexCrs = GeometryUtils.getIndexCrs(outputPrimaryIndex);
outputCrsCode = GeometryUtils.getCrsCode(outputIndexCrs);
} else {
final SpatialDimensionalityTypeProvider sdp = new SpatialDimensionalityTypeProvider();
final SpatialOptions so = sdp.createOptions();
so.setCrs(inputCrsCode);
outputPrimaryIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(so);
outputIndexCrs = inputIndexCrs;
outputCrsCode = inputCrsCode;
}
final CoordinateSystem cs = outputIndexCrs.getCoordinateSystem();
final CoordinateSystemAxis csx = cs.getAxis(0);
final CoordinateSystemAxis csy = cs.getAxis(1);
final double xMax = csx.getMaximumValue();
final double xMin = csx.getMinimumValue();
final double yMax = csy.getMaximumValue();
final double yMin = csy.getMinimumValue();
if ((xMax == Double.POSITIVE_INFINITY) || (xMin == Double.NEGATIVE_INFINITY) || (yMax == Double.POSITIVE_INFINITY) || (yMin == Double.NEGATIVE_INFINITY)) {
LOGGER.error("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
throw new RuntimeException("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
}
if (cqlFilter != null) {
bldr.constraints(bldr.constraintsFactory().cqlConstraints(cqlFilter));
}
// Load RDD from datastore
final RDDOptions kdeOpts = new RDDOptions();
kdeOpts.setMinSplits(minSplits);
kdeOpts.setMaxSplits(maxSplits);
kdeOpts.setQuery(bldr.build());
final Function<Double, Double> identity = x -> x;
final Function2<Double, Double, Double> sum = (final Double x, final Double y) -> {
return x + y;
};
final RasterDataAdapter adapter = RasterUtils.createDataAdapterTypeDouble(coverageName, KDEReducer.NUM_BANDS, tileSize, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, new NoDataMergeStrategy());
outputDataStore.createDataStore().addType(adapter, outputPrimaryIndex);
// The following "inner" variables are created to give access to member
// variables within lambda
// expressions
// tileSize;
final int innerTileSize = 1;
final String innerCoverageName = coverageName;
for (int level = minLevel; level <= maxLevel; level++) {
final int numXTiles = (int) Math.pow(2, level + 1);
final int numYTiles = (int) Math.pow(2, level);
// * tileSize;
final int numXPosts = numXTiles;
// * tileSize;
final int numYPosts = numYTiles;
final GeoWaveRDD kdeRDD = GeoWaveRDDLoader.loadRDD(session.sparkContext(), inputDataStore, kdeOpts);
JavaPairRDD<Double, Long> cells = kdeRDD.getRawRDD().flatMapToPair(new GeoWaveCellMapper(numXPosts, numYPosts, xMin, xMax, yMin, yMax, inputCrsCode, outputCrsCode)).combineByKey(identity, sum, sum).mapToPair(item -> item.swap());
cells = cells.partitionBy(new RangePartitioner(cells.getNumPartitions(), cells.rdd(), true, scala.math.Ordering.Double$.MODULE$, scala.reflect.ClassTag$.MODULE$.apply(Double.class))).sortByKey(false).cache();
final long count = cells.count();
if (count == 0) {
LOGGER.warn("No cells produced by KDE");
continue;
}
final double max = cells.first()._1;
JavaRDD<GridCoverage> rdd = cells.zipWithIndex().map(t -> {
final TileInfo tileInfo = fromCellIndexToTileInfo(t._1._2, numXPosts, numYPosts, numXTiles, numYTiles, xMin, xMax, yMin, yMax, innerTileSize);
final WritableRaster raster = RasterUtils.createRasterTypeDouble(NUM_BANDS, innerTileSize);
final double normalizedValue = t._1._1 / max;
// because we are using a Double as the key, the ordering
// isn't always completely reproducible as Double equals does not
// take into account an epsilon
final double percentile = (count - t._2) / ((double) count);
raster.setSample(tileInfo.x, tileInfo.y, 0, t._1._1);
raster.setSample(tileInfo.x, tileInfo.y, 1, normalizedValue);
raster.setSample(tileInfo.x, tileInfo.y, 2, percentile);
return RasterUtils.createCoverageTypeDouble(innerCoverageName, tileInfo.tileWestLon, tileInfo.tileEastLon, tileInfo.tileSouthLat, tileInfo.tileNorthLat, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, raster, GeometryUtils.DEFAULT_CRS_STR);
});
LOGGER.debug("Writing results to output store...");
if (tileSize > 1) {
// byte[] adapterBytes = PersistenceUtils.toBinary(adapter);
// byte[] indexBytes = PersistenceUtils.toBinary(outputPrimaryIndex);
rdd = rdd.flatMapToPair(new TransformTileSize(adapter, outputPrimaryIndex)).groupByKey().map(new MergeOverlappingTiles(adapter, outputPrimaryIndex));
}
RDDUtils.writeRasterToGeoWave(jsc.sc(), outputPrimaryIndex, outputDataStore, adapter, rdd);
LOGGER.debug("Results successfully written!");
}
}
use of org.locationtech.geowave.core.geotime.index.SpatialDimensionalityTypeProvider in project geowave by locationtech.
the class TestUtils method createWebMercatorSpatialIndex.
public static Index createWebMercatorSpatialIndex() {
final SpatialDimensionalityTypeProvider sdp = new SpatialDimensionalityTypeProvider();
final SpatialOptions so = sdp.createOptions();
so.setCrs(CUSTOM_CRSCODE);
final Index primaryIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(so);
return primaryIndex;
}
use of org.locationtech.geowave.core.geotime.index.SpatialDimensionalityTypeProvider in project geowave by locationtech.
the class KDEJobRunner method runJob.
/**
* Main method to execute the MapReduce analytic.
*/
@SuppressWarnings("deprecation")
public int runJob() throws Exception {
Configuration conf = super.getConf();
if (conf == null) {
conf = new Configuration();
setConf(conf);
}
Index inputPrimaryIndex = null;
final Index[] idxArray = inputDataStoreOptions.createDataStore().getIndices();
for (final Index idx : idxArray) {
if ((idx != null) && ((kdeCommandLineOptions.getIndexName() == null) || kdeCommandLineOptions.getIndexName().equals(idx.getName()))) {
inputPrimaryIndex = idx;
break;
}
}
final CoordinateReferenceSystem inputIndexCrs = GeometryUtils.getIndexCrs(inputPrimaryIndex);
final String inputCrsCode = GeometryUtils.getCrsCode(inputIndexCrs);
Index outputPrimaryIndex = outputIndex;
CoordinateReferenceSystem outputIndexCrs = null;
String outputCrsCode = null;
if (outputPrimaryIndex != null) {
outputIndexCrs = GeometryUtils.getIndexCrs(outputPrimaryIndex);
outputCrsCode = GeometryUtils.getCrsCode(outputIndexCrs);
} else {
final SpatialDimensionalityTypeProvider sdp = new SpatialDimensionalityTypeProvider();
final SpatialOptions so = sdp.createOptions();
so.setCrs(inputCrsCode);
outputPrimaryIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(so);
outputIndexCrs = inputIndexCrs;
outputCrsCode = inputCrsCode;
}
final CoordinateSystem cs = outputIndexCrs.getCoordinateSystem();
final CoordinateSystemAxis csx = cs.getAxis(0);
final CoordinateSystemAxis csy = cs.getAxis(1);
final double xMax = csx.getMaximumValue();
final double xMin = csx.getMinimumValue();
final double yMax = csy.getMaximumValue();
final double yMin = csy.getMinimumValue();
if ((xMax == Double.POSITIVE_INFINITY) || (xMin == Double.NEGATIVE_INFINITY) || (yMax == Double.POSITIVE_INFINITY) || (yMin == Double.NEGATIVE_INFINITY)) {
LOGGER.error("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
throw new RuntimeException("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
}
DataStorePluginOptions rasterResizeOutputDataStoreOptions;
String kdeCoverageName;
// the KDE output and then run a resize operation
if ((kdeCommandLineOptions.getTileSize() > 1)) {
// this is the ending data store options after resize, the KDE will
// need to output to a temporary namespace, a resize operation
// will use the outputDataStoreOptions
rasterResizeOutputDataStoreOptions = outputDataStoreOptions;
// first clone the outputDataStoreOptions, then set it to a tmp
// namespace
final Map<String, String> configOptions = outputDataStoreOptions.getOptionsAsMap();
final StoreFactoryOptions options = ConfigUtils.populateOptionsFromList(outputDataStoreOptions.getFactoryFamily().getDataStoreFactory().createOptionsInstance(), configOptions);
options.setGeoWaveNamespace(outputDataStoreOptions.getGeoWaveNamespace() + "_tmp");
outputDataStoreOptions = new DataStorePluginOptions(options);
kdeCoverageName = kdeCommandLineOptions.getCoverageName() + TMP_COVERAGE_SUFFIX;
} else {
rasterResizeOutputDataStoreOptions = null;
kdeCoverageName = kdeCommandLineOptions.getCoverageName();
}
if (kdeCommandLineOptions.getHdfsHostPort() == null) {
final Properties configProperties = ConfigOptions.loadProperties(configFile);
final String hdfsFSUrl = ConfigHDFSCommand.getHdfsUrl(configProperties);
kdeCommandLineOptions.setHdfsHostPort(hdfsFSUrl);
}
GeoWaveConfiguratorBase.setRemoteInvocationParams(kdeCommandLineOptions.getHdfsHostPort(), kdeCommandLineOptions.getJobTrackerOrResourceManHostPort(), conf);
conf.setInt(MAX_LEVEL_KEY, kdeCommandLineOptions.getMaxLevel());
conf.setInt(MIN_LEVEL_KEY, kdeCommandLineOptions.getMinLevel());
conf.set(COVERAGE_NAME_KEY, kdeCoverageName);
if (kdeCommandLineOptions.getCqlFilter() != null) {
conf.set(GaussianCellMapper.CQL_FILTER_KEY, kdeCommandLineOptions.getCqlFilter());
}
conf.setDouble(X_MIN_KEY, xMin);
conf.setDouble(X_MAX_KEY, xMax);
conf.setDouble(Y_MIN_KEY, yMin);
conf.setDouble(Y_MAX_KEY, yMax);
conf.set(INPUT_CRSCODE_KEY, inputCrsCode);
conf.set(OUTPUT_CRSCODE_KEY, outputCrsCode);
preJob1Setup(conf);
final Job job = new Job(conf);
job.setJarByClass(this.getClass());
addJobClasspathDependencies(job, conf);
job.setJobName(getJob1Name());
job.setMapperClass(getJob1Mapper());
job.setCombinerClass(CellSummationCombiner.class);
job.setReducerClass(getJob1Reducer());
job.setMapOutputKeyClass(LongWritable.class);
job.setMapOutputValueClass(DoubleWritable.class);
job.setOutputKeyClass(DoubleWritable.class);
job.setOutputValueClass(LongWritable.class);
job.setInputFormatClass(GeoWaveInputFormat.class);
job.setOutputFormatClass(SequenceFileOutputFormat.class);
job.setNumReduceTasks(8);
job.setSpeculativeExecution(false);
final PersistentAdapterStore adapterStore = inputDataStoreOptions.createAdapterStore();
final IndexStore indexStore = inputDataStoreOptions.createIndexStore();
final InternalAdapterStore internalAdapterStore = inputDataStoreOptions.createInternalAdapterStore();
final short internalAdapterId = internalAdapterStore.getAdapterId(kdeCommandLineOptions.getFeatureType());
final DataTypeAdapter<?> adapter = adapterStore.getAdapter(internalAdapterId).getAdapter();
VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder().addTypeName(adapter.getTypeName());
if (kdeCommandLineOptions.getIndexName() != null) {
bldr = bldr.indexName(kdeCommandLineOptions.getIndexName());
}
GeoWaveInputFormat.setMinimumSplitCount(job.getConfiguration(), kdeCommandLineOptions.getMinSplits());
GeoWaveInputFormat.setMaximumSplitCount(job.getConfiguration(), kdeCommandLineOptions.getMaxSplits());
GeoWaveInputFormat.setStoreOptions(job.getConfiguration(), inputDataStoreOptions);
if (kdeCommandLineOptions.getCqlFilter() != null) {
Geometry bbox = null;
if (adapter instanceof GeotoolsFeatureDataAdapter) {
final String geometryAttribute = ((GeotoolsFeatureDataAdapter) adapter).getFeatureType().getGeometryDescriptor().getLocalName();
final Filter filter = ECQL.toFilter(kdeCommandLineOptions.getCqlFilter());
final ExtractGeometryFilterVisitorResult geoAndCompareOpData = (ExtractGeometryFilterVisitorResult) filter.accept(new ExtractGeometryFilterVisitor(GeometryUtils.getDefaultCRS(), geometryAttribute), null);
bbox = geoAndCompareOpData.getGeometry();
}
if ((bbox != null) && !bbox.equals(GeometryUtils.infinity())) {
bldr = bldr.constraints(bldr.constraintsFactory().spatialTemporalConstraints().spatialConstraints(bbox).build());
}
}
GeoWaveInputFormat.setQuery(conf, bldr.build(), adapterStore, internalAdapterStore, indexStore);
FileSystem fs = null;
try {
fs = FileSystem.get(conf);
fs.delete(new Path("/tmp/" + inputDataStoreOptions.getGeoWaveNamespace() + "_stats_" + kdeCommandLineOptions.getMinLevel() + "_" + kdeCommandLineOptions.getMaxLevel() + "_" + kdeCommandLineOptions.getCoverageName()), true);
FileOutputFormat.setOutputPath(job, new Path("/tmp/" + inputDataStoreOptions.getGeoWaveNamespace() + "_stats_" + kdeCommandLineOptions.getMinLevel() + "_" + kdeCommandLineOptions.getMaxLevel() + "_" + kdeCommandLineOptions.getCoverageName() + "/basic"));
final boolean job1Success = job.waitForCompletion(true);
boolean job2Success = false;
boolean postJob2Success = false;
// Linear MapReduce job chaining
if (job1Success) {
setupEntriesPerLevel(job, conf);
// Stats Reducer Job configuration parameters
final Job statsReducer = new Job(conf);
statsReducer.setJarByClass(this.getClass());
addJobClasspathDependencies(statsReducer, conf);
statsReducer.setJobName(getJob2Name());
statsReducer.setMapperClass(IdentityMapper.class);
statsReducer.setPartitionerClass(getJob2Partitioner());
statsReducer.setReducerClass(getJob2Reducer());
statsReducer.setNumReduceTasks(getJob2NumReducers((kdeCommandLineOptions.getMaxLevel() - kdeCommandLineOptions.getMinLevel()) + 1));
statsReducer.setMapOutputKeyClass(DoubleWritable.class);
statsReducer.setMapOutputValueClass(LongWritable.class);
statsReducer.setOutputKeyClass(getJob2OutputKeyClass());
statsReducer.setOutputValueClass(getJob2OutputValueClass());
statsReducer.setInputFormatClass(SequenceFileInputFormat.class);
statsReducer.setOutputFormatClass(getJob2OutputFormatClass());
FileInputFormat.setInputPaths(statsReducer, new Path("/tmp/" + inputDataStoreOptions.getGeoWaveNamespace() + "_stats_" + kdeCommandLineOptions.getMinLevel() + "_" + kdeCommandLineOptions.getMaxLevel() + "_" + kdeCommandLineOptions.getCoverageName() + "/basic"));
setupJob2Output(conf, statsReducer, outputDataStoreOptions.getGeoWaveNamespace(), kdeCoverageName, outputPrimaryIndex);
job2Success = statsReducer.waitForCompletion(true);
if (job2Success) {
postJob2Success = postJob2Actions(conf, outputDataStoreOptions.getGeoWaveNamespace(), kdeCoverageName);
}
} else {
job2Success = false;
}
if (rasterResizeOutputDataStoreOptions != null) {
// delegate to resize command to wrap it up with the correctly
// requested tile size
final ResizeMRCommand resizeCommand = new ResizeMRCommand();
final File configFile = File.createTempFile("temp-config", null);
final ManualOperationParams params = new ManualOperationParams();
params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
final AddStoreCommand addStore = new AddStoreCommand();
addStore.setParameters("temp-out");
addStore.setPluginOptions(outputDataStoreOptions);
addStore.execute(params);
addStore.setParameters("temp-raster-out");
addStore.setPluginOptions(rasterResizeOutputDataStoreOptions);
addStore.execute(params);
// We're going to override these anyway.
resizeCommand.setParameters("temp-out", "temp-raster-out");
resizeCommand.getOptions().setInputCoverageName(kdeCoverageName);
resizeCommand.getOptions().setMinSplits(kdeCommandLineOptions.getMinSplits());
resizeCommand.getOptions().setMaxSplits(kdeCommandLineOptions.getMaxSplits());
resizeCommand.setHdfsHostPort(kdeCommandLineOptions.getHdfsHostPort());
resizeCommand.setJobTrackerOrResourceManHostPort(kdeCommandLineOptions.getJobTrackerOrResourceManHostPort());
resizeCommand.getOptions().setOutputCoverageName(kdeCommandLineOptions.getCoverageName());
resizeCommand.getOptions().setOutputTileSize(kdeCommandLineOptions.getTileSize());
final int resizeStatus = ToolRunner.run(resizeCommand.createRunner(params), new String[] {});
if (resizeStatus == 0) {
// delegate to clear command to clean up with tmp namespace
// after successful resize
final ClearStoreCommand clearCommand = new ClearStoreCommand();
clearCommand.setParameters("temp-out");
clearCommand.execute(params);
} else {
LOGGER.warn("Resize command error code '" + resizeStatus + "'. Retaining temporary namespace '" + outputDataStoreOptions.getGeoWaveNamespace() + "' with tile size of 1.");
}
}
fs.delete(new Path("/tmp/" + inputDataStoreOptions.getGeoWaveNamespace() + "_stats_" + kdeCommandLineOptions.getMinLevel() + "_" + kdeCommandLineOptions.getMaxLevel() + "_" + kdeCommandLineOptions.getCoverageName()), true);
return (job1Success && job2Success && postJob2Success) ? 0 : 1;
} finally {
if (fs != null) {
try {
fs.close();
} catch (final IOException e) {
LOGGER.info(e.getMessage());
// Attempt to close, but don't throw an error if it is
// already closed.
// Log message, so find bugs does not complain.
}
}
}
}
Aggregations