Examples with GeoWaveOutputKey org.locationtech.geowave.mapreduce.output.GeoWaveOutputKey used on opensource projects

Example 1 with GeoWaveOutputKey

use of org.locationtech.geowave.mapreduce.output.GeoWaveOutputKey in project geowave by locationtech.

the class RDDUtils method writeToGeoWave.

/**
 * Translate a set of objects in a JavaRDD to a provided type and push to GeoWave
 *
 * @throws IOException
 */
private static void writeToGeoWave(final SparkContext sc, final Index index, final DataStorePluginOptions outputStoreOptions, final DataTypeAdapter adapter, final JavaRDD<SimpleFeature> inputRDD) throws IOException {
    // setup the configuration and the output format
    final Configuration conf = new org.apache.hadoop.conf.Configuration(sc.hadoopConfiguration());
    GeoWaveOutputFormat.setStoreOptions(conf, outputStoreOptions);
    GeoWaveOutputFormat.addIndex(conf, index);
    GeoWaveOutputFormat.addDataAdapter(conf, adapter);
    // create the job
    final Job job = new Job(conf);
    job.setOutputKeyClass(GeoWaveOutputKey.class);
    job.setOutputValueClass(SimpleFeature.class);
    job.setOutputFormatClass(GeoWaveOutputFormat.class);
    // broadcast string names
    final ClassTag<String> stringTag = scala.reflect.ClassTag$.MODULE$.apply(String.class);
    final Broadcast<String> typeName = sc.broadcast(adapter.getTypeName(), stringTag);
    final Broadcast<String> indexName = sc.broadcast(index.getName(), stringTag);
    // map to a pair containing the output key and the output value
    inputRDD.mapToPair(feat -> new Tuple2<>(new GeoWaveOutputKey(typeName.value(), indexName.value()), feat)).saveAsNewAPIHadoopDataset(job.getConfiguration());
}

Example 2 with GeoWaveOutputKey

use of org.locationtech.geowave.mapreduce.output.GeoWaveOutputKey in project geowave by locationtech.

the class InputToOutputKeyReducer method setup.

@Override
protected void setup(final Reducer<GeoWaveInputKey, ObjectWritable, GeoWaveOutputKey, Object>.Context context) throws IOException, InterruptedException {
    super.setup(context);
    internalAdapterStore = GeoWaveOutputFormat.getJobContextInternalAdapterStore(context);
    final ScopedJobConfiguration config = new ScopedJobConfiguration(context.getConfiguration(), InputToOutputKeyReducer.class, LOGGER);
    outputKey = new GeoWaveOutputKey("na", new String[] { config.getString(OutputParameters.Output.INDEX_ID, "na") });
}

Example 3 with GeoWaveOutputKey

use of org.locationtech.geowave.mapreduce.output.GeoWaveOutputKey in project geowave by locationtech.

the class OSMConversionMapper method map.

@Override
protected void map(final Key key, final Value value, final Context context) throws IOException, InterruptedException {
    final List<SimpleFeature> sf = sfg.mapOSMtoSimpleFeature(WholeRowIterator.decodeRow(key, value), osmProvider);
    if ((sf != null) && (sf.size() > 0)) {
        for (final SimpleFeature feat : sf) {
            final String name = feat.getType().getTypeName();
            context.write(new GeoWaveOutputKey(name, indexName), feat);
        }
    }
}

Example 4 with GeoWaveOutputKey

use of org.locationtech.geowave.mapreduce.output.GeoWaveOutputKey in project geowave by locationtech.

the class KDEReducer method reduce.

@Override
protected void reduce(final DoubleWritable key, final Iterable<LongWritable> values, final Context context) throws IOException, InterruptedException {
    if (key.get() < 0) {
        final double prevMax = -key.get();
        if (prevMax > max) {
            max = prevMax;
        }
    } else {
        final double value = key.get();
        final double normalizedValue = value / max;
        // for consistency give all cells with matching weight the same
        // percentile
        // because we are using a DoubleWritable as the key, the ordering
        // isn't always completely reproducible as Double equals does not
        // take into account an epsilon, but we can make it reproducible by
        // doing a comparison with the previous value using an appropriate
        // epsilon
        final double percentile;
        if (FloatCompareUtils.checkDoublesEqual(prevValue, value, WEIGHT_EPSILON)) {
            percentile = prevPct;
        } else {
            percentile = (currentKey + 1.0) / totalKeys;
            prevPct = percentile;
            prevValue = value;
        }
        // calculate weights for this key
        for (final LongWritable v : values) {
            final long cellIndex = v.get() / numLevels;
            final TileInfo tileInfo = fromCellIndexToTileInfo(cellIndex);
            final WritableRaster raster = RasterUtils.createRasterTypeDouble(NUM_BANDS, KDEJobRunner.TILE_SIZE);
            raster.setSample(tileInfo.x, tileInfo.y, 0, key.get());
            raster.setSample(tileInfo.x, tileInfo.y, 1, normalizedValue);
            raster.setSample(tileInfo.x, tileInfo.y, 2, percentile);
            context.write(new GeoWaveOutputKey(coverageName, indexList.toArray(new String[0])), RasterUtils.createCoverageTypeDouble(coverageName, tileInfo.tileWestLon, tileInfo.tileEastLon, tileInfo.tileSouthLat, tileInfo.tileNorthLat, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, raster, crsCode));
            currentKey++;
        }
    }
}

Example 5 with GeoWaveOutputKey

use of org.locationtech.geowave.mapreduce.output.GeoWaveOutputKey in project geowave by locationtech.

the class ComparisonAccumuloStatsReducer method reduce.

@Override
protected void reduce(final ComparisonCellData key, final Iterable<LongWritable> values, final Context context) throws IOException, InterruptedException {
    // for consistency give all cells with matching weight the same
    // percentile
    final double percentile = (currentKey + 1.0) / totalKeys;
    // calculate weights for this key
    for (final LongWritable v : values) {
        final long cellIndex = v.get() / numLevels;
        final Point2d[] bbox = fromIndexToLL_UR(cellIndex);
        final WritableRaster raster = RasterUtils.createRasterTypeDouble(NUM_BANDS, TILE_SIZE);
        raster.setSample(0, 0, 0, key.getSummerPercentile());
        raster.setSample(0, 0, 1, key.getWinterPercentile());
        raster.setSample(0, 0, 2, key.getCombinedPercentile());
        raster.setSample(0, 0, 3, percentile);
        context.write(new GeoWaveOutputKey(coverageName, indexNames), RasterUtils.createCoverageTypeDouble(coverageName, bbox[0].x, bbox[1].x, bbox[0].y, bbox[1].y, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, raster));
        currentKey++;
    }
}