Examples with GeoWaveInputKey org.locationtech.geowave.mapreduce.input.GeoWaveInputKey used on opensource projects

Example 1 with GeoWaveInputKey

use of org.locationtech.geowave.mapreduce.input.GeoWaveInputKey in project geowave by locationtech.

the class RasterTileResizeSparkRunner method run.

public void run() throws IOException {
    initContext();
    // Validate inputs
    if (inputStoreOptions == null) {
        LOGGER.error("You must supply an input datastore!");
        throw new IOException("You must supply an input datastore!");
    }
    final InternalAdapterStore internalAdapterStore = inputStoreOptions.createInternalAdapterStore();
    final short internalAdapterId = internalAdapterStore.getAdapterId(rasterResizeOptions.getInputCoverageName());
    final DataTypeAdapter adapter = inputStoreOptions.createAdapterStore().getAdapter(internalAdapterId).getAdapter();
    if (adapter == null) {
        throw new IllegalArgumentException("Adapter for coverage '" + rasterResizeOptions.getInputCoverageName() + "' does not exist in namespace '" + inputStoreOptions.getGeoWaveNamespace() + "'");
    }
    Index index = null;
    final IndexStore indexStore = inputStoreOptions.createIndexStore();
    if (rasterResizeOptions.getIndexName() != null) {
        index = indexStore.getIndex(rasterResizeOptions.getIndexName());
    }
    if (index == null) {
        try (CloseableIterator<Index> indices = indexStore.getIndices()) {
            index = indices.next();
        }
        if (index == null) {
            throw new IllegalArgumentException("Index does not exist in namespace '" + inputStoreOptions.getGeoWaveNamespace() + "'");
        }
    }
    final RasterDataAdapter newAdapter = new RasterDataAdapter((RasterDataAdapter) adapter, rasterResizeOptions.getOutputCoverageName(), rasterResizeOptions.getOutputTileSize());
    final DataStore store = outputStoreOptions.createDataStore();
    store.addType(newAdapter, index);
    final short newInternalAdapterId = outputStoreOptions.createInternalAdapterStore().addTypeName(newAdapter.getTypeName());
    final RDDOptions options = new RDDOptions();
    if (rasterResizeOptions.getMinSplits() != null) {
        options.setMinSplits(rasterResizeOptions.getMinSplits());
    }
    if (rasterResizeOptions.getMaxSplits() != null) {
        options.setMaxSplits(rasterResizeOptions.getMaxSplits());
    }
    final JavaPairRDD<GeoWaveInputKey, GridCoverage> inputRDD = GeoWaveRDDLoader.loadRawRasterRDD(jsc.sc(), inputStoreOptions, index.getName(), rasterResizeOptions.getMinSplits(), rasterResizeOptions.getMaxSplits());
    LOGGER.debug("Writing results to output store...");
    RDDUtils.writeRasterToGeoWave(jsc.sc(), index, outputStoreOptions, newAdapter, inputRDD.flatMapToPair(new RasterResizeMappingFunction(internalAdapterId, newInternalAdapterId, newAdapter, index)).groupByKey().map(new MergeRasterFunction(internalAdapterId, newInternalAdapterId, newAdapter, index)));
    LOGGER.debug("Results successfully written!");
}

Example 2 with GeoWaveInputKey

use of org.locationtech.geowave.mapreduce.input.GeoWaveInputKey in project geowave by locationtech.

the class GeoWaveRDDLoader method loadRawRDD.

public static JavaPairRDD<GeoWaveInputKey, SimpleFeature> loadRawRDD(final SparkContext sc, final DataStorePluginOptions storeOptions, final RDDOptions rddOpts) throws IOException {
    if (sc == null) {
        LOGGER.error("Must supply a valid Spark Context. Please set SparkContext and try again.");
        return null;
    }
    if (storeOptions == null) {
        LOGGER.error("Must supply input store to load. Please set storeOptions and try again.");
        return null;
    }
    if (rddOpts == null) {
        LOGGER.error("Must supply valid RDDOptions to load a rdd.");
        return null;
    }
    final Configuration conf = new Configuration(sc.hadoopConfiguration());
    GeoWaveInputFormat.setStoreOptions(conf, storeOptions);
    if (rddOpts.getQuery() != null) {
        GeoWaveInputFormat.setQuery(conf, rddOpts.getQuery(), storeOptions.createAdapterStore(), storeOptions.createInternalAdapterStore(), storeOptions.createIndexStore());
    }
    if ((rddOpts.getMinSplits() > -1) || (rddOpts.getMaxSplits() > -1)) {
        GeoWaveInputFormat.setMinimumSplitCount(conf, rddOpts.getMinSplits());
        GeoWaveInputFormat.setMaximumSplitCount(conf, rddOpts.getMaxSplits());
    } else {
        final int defaultSplitsSpark = sc.getConf().getInt("spark.default.parallelism", -1);
        // Otherwise just fallback to default according to index strategy
        if (defaultSplitsSpark != -1) {
            GeoWaveInputFormat.setMinimumSplitCount(conf, defaultSplitsSpark);
            GeoWaveInputFormat.setMaximumSplitCount(conf, defaultSplitsSpark);
        }
    }
    final RDD<Tuple2<GeoWaveInputKey, SimpleFeature>> rdd = sc.newAPIHadoopRDD(conf, GeoWaveInputFormat.class, GeoWaveInputKey.class, SimpleFeature.class);
    final JavaPairRDD<GeoWaveInputKey, SimpleFeature> javaRdd = JavaPairRDD.fromJavaRDD(rdd.toJavaRDD());
    return javaRdd;
}

Example 3 with GeoWaveInputKey

use of org.locationtech.geowave.mapreduce.input.GeoWaveInputKey in project geowave by locationtech.

the class GeoWaveRDDLoader method loadRawRasterRDD.

public static JavaPairRDD<GeoWaveInputKey, GridCoverage> loadRawRasterRDD(final SparkContext sc, final DataStorePluginOptions storeOptions, final String indexName, final Integer minSplits, final Integer maxSplits) throws IOException {
    if (sc == null) {
        LOGGER.error("Must supply a valid Spark Context. Please set SparkContext and try again.");
        return null;
    }
    if (storeOptions == null) {
        LOGGER.error("Must supply input store to load. Please set storeOptions and try again.");
        return null;
    }
    final Configuration conf = new Configuration(sc.hadoopConfiguration());
    GeoWaveInputFormat.setStoreOptions(conf, storeOptions);
    if (indexName != null) {
        GeoWaveInputFormat.setQuery(conf, QueryBuilder.newBuilder().indexName(indexName).build(), storeOptions.createAdapterStore(), storeOptions.createInternalAdapterStore(), storeOptions.createIndexStore());
    }
    if (((minSplits != null) && (minSplits > -1)) || ((maxSplits != null) && (maxSplits > -1))) {
        GeoWaveInputFormat.setMinimumSplitCount(conf, minSplits);
        GeoWaveInputFormat.setMaximumSplitCount(conf, maxSplits);
    } else {
        final int defaultSplitsSpark = sc.getConf().getInt("spark.default.parallelism", -1);
        // Otherwise just fallback to default according to index strategy
        if (defaultSplitsSpark != -1) {
            GeoWaveInputFormat.setMinimumSplitCount(conf, defaultSplitsSpark);
            GeoWaveInputFormat.setMaximumSplitCount(conf, defaultSplitsSpark);
        }
    }
    final RDD<Tuple2<GeoWaveInputKey, GridCoverage>> rdd = sc.newAPIHadoopRDD(conf, GeoWaveInputFormat.class, GeoWaveInputKey.class, GridCoverage.class);
    final JavaPairRDD<GeoWaveInputKey, GridCoverage> javaRdd = JavaPairRDD.fromJavaRDD(rdd.toJavaRDD());
    return javaRdd;
}

Example 4 with GeoWaveInputKey

use of org.locationtech.geowave.mapreduce.input.GeoWaveInputKey in project geowave by locationtech.

the class DBScanMapReduceTest method test8With4.

@Test
public void test8With4() throws IOException {
    final Random r = new Random(3434);
    for (int i = 0; i < 8; i++) {
        final SimpleFeature feature = createTestFeature("f" + i, new Coordinate(round(30.0 + (r.nextGaussian() * 0.00001)), round(30.0 + (r.nextGaussian() * 0.00001))));
        mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature.getID())), feature);
    }
    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults = mapDriver.run();
    final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions = getReducerDataFromMapperInput(mapperResults);
    reduceDriver.addAll(partitions);
    reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE), 4);
    final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
    assertEquals(1, reduceResults.size());
}

Example 5 with GeoWaveInputKey

use of org.locationtech.geowave.mapreduce.input.GeoWaveInputKey in project geowave by locationtech.

the class DBScanMapReduceTest method testReducer.

@Test
public void testReducer() throws IOException {
    final SimpleFeature feature1 = createTestFeature("f1", new Coordinate(30.0, 30.00000001));
    final SimpleFeature feature2 = createTestFeature("f2", new Coordinate(50.001, 50.001));
    final SimpleFeature feature3 = createTestFeature("f3", new Coordinate(30.00000001, 30.00000001));
    final SimpleFeature feature4 = createTestFeature("f4", new Coordinate(50.0011, 50.00105));
    final SimpleFeature feature5 = createTestFeature("f5", new Coordinate(50.00112, 50.00111));
    final SimpleFeature feature6 = createTestFeature("f6", new Coordinate(30.00000001, 30.00000002));
    final SimpleFeature feature7 = createTestFeature("f7", new Coordinate(50.00113, 50.00114));
    final SimpleFeature feature8 = createTestFeature("f8", new Coordinate(40.00000001, 40.000000002));
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature1.getID())), feature1);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature2.getID())), feature2);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature3.getID())), feature3);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature4.getID())), feature4);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature5.getID())), feature5);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature6.getID())), feature6);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature7.getID())), feature7);
    mapDriver.addInput(new GeoWaveInputKey(adapterId, new ByteArray(feature8.getID())), feature8);
    final List<Pair<PartitionDataWritable, AdapterWithObjectWritable>> mapperResults = mapDriver.run();
    assertNotNull(getPartitionDataFor(mapperResults, feature1.getID(), true));
    assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
    assertNotNull(getPartitionDataFor(mapperResults, feature2.getID(), true));
    assertNotNull(getPartitionDataFor(mapperResults, feature3.getID(), true));
    assertEquals(getPartitionDataFor(mapperResults, feature1.getID(), true).getCompositeKey(), getPartitionDataFor(mapperResults, feature3.getID(), true).getCompositeKey());
    assertEquals(getPartitionDataFor(mapperResults, feature6.getID(), true).getCompositeKey(), getPartitionDataFor(mapperResults, feature3.getID(), true).getCompositeKey());
    assertEquals(getPartitionDataFor(mapperResults, feature5.getID(), true).getCompositeKey(), getPartitionDataFor(mapperResults, feature7.getID(), true).getCompositeKey());
    assertEquals(getPartitionDataFor(mapperResults, feature5.getID(), true).getCompositeKey(), getPartitionDataFor(mapperResults, feature4.getID(), true).getCompositeKey());
    final List<Pair<PartitionDataWritable, List<AdapterWithObjectWritable>>> partitions = getReducerDataFromMapperInput(mapperResults);
    reduceDriver.addAll(partitions);
    reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(NNMapReduce.class, ClusteringParameters.Clustering.MINIMUM_SIZE), 2);
    final List<Pair<GeoWaveInputKey, ObjectWritable>> reduceResults = reduceDriver.run();
    assertEquals(2, reduceResults.size());
/*
     * assertEquals( feature3.getID(), find( reduceResults, feature1.getID()).toString());
     *
     * assertEquals( feature1.getID(), find( reduceResults, feature3.getID()).toString());
     *
     * assertEquals( feature4.getID(), find( reduceResults, feature2.getID()).toString());
     *
     * assertEquals( feature2.getID(), find( reduceResults, feature4.getID()).toString());
     */
}