Examples with SpatialOptions org.locationtech.geowave.core.geotime.index.SpatialOptions used on opensource projects

Example 1 with SpatialOptions

use of org.locationtech.geowave.core.geotime.index.SpatialOptions in project geowave by locationtech.

the class KMeansUtils method writeClusterHulls.

public static DataTypeAdapter writeClusterHulls(final JavaRDD<Vector> inputCentroids, final KMeansModel clusterModel, final DataStorePluginOptions outputDataStore, final String hullAdapterName, final boolean computeMetadata) {
    final JavaPairRDD<Integer, Iterable<Vector>> groupByRdd = KMeansHullGenerator.groupByIndex(inputCentroids, clusterModel);
    final JavaPairRDD<Integer, Geometry> hullRdd = KMeansHullGenerator.generateHullsRDD(groupByRdd);
    final SimpleFeatureTypeBuilder typeBuilder = new SimpleFeatureTypeBuilder();
    typeBuilder.setName(hullAdapterName);
    typeBuilder.setNamespaceURI(BasicFeatureTypes.DEFAULT_NAMESPACE);
    try {
        typeBuilder.setCRS(CRS.decode("EPSG:4326", true));
    } catch (final FactoryException e) {
        LOGGER.error(e.getMessage(), e);
    }
    final AttributeTypeBuilder attrBuilder = new AttributeTypeBuilder();
    typeBuilder.add(attrBuilder.binding(Geometry.class).nillable(false).buildDescriptor(Geometry.class.getName().toString()));
    typeBuilder.add(attrBuilder.binding(Integer.class).nillable(false).buildDescriptor("ClusterIndex"));
    typeBuilder.add(attrBuilder.binding(Integer.class).nillable(false).buildDescriptor("Count"));
    typeBuilder.add(attrBuilder.binding(Double.class).nillable(false).buildDescriptor("Area"));
    typeBuilder.add(attrBuilder.binding(Double.class).nillable(false).buildDescriptor("Density"));
    final SimpleFeatureType sfType = typeBuilder.buildFeatureType();
    final SimpleFeatureBuilder sfBuilder = new SimpleFeatureBuilder(sfType);
    final FeatureDataAdapter featureAdapter = new FeatureDataAdapter(sfType);
    final DataStore featureStore = outputDataStore.createDataStore();
    final Index featureIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final PolygonAreaCalculator polyCalc = (computeMetadata ? new PolygonAreaCalculator() : null);
    featureStore.addType(featureAdapter, featureIndex);
    try (Writer writer = featureStore.createWriter(featureAdapter.getTypeName())) {
        for (final Tuple2<Integer, Geometry> hull : hullRdd.collect()) {
            final Integer index = hull._1;
            final Geometry geom = hull._2;
            sfBuilder.set(Geometry.class.getName(), geom);
            sfBuilder.set("ClusterIndex", index);
            int count = 0;
            double area = 0.0;
            double density = 0.0;
            if (computeMetadata) {
                for (final Iterable<Vector> points : groupByRdd.lookup(index)) {
                    final Vector[] pointVec = Iterables.toArray(points, Vector.class);
                    count += pointVec.length;
                }
                try {
                    // HP Fortify "NULL Pointer Dereference" false positive
                    // Exception handling will catch if polyCalc is null
                    area = polyCalc.getAreaDensify(geom);
                    density = count / area;
                } catch (final Exception e) {
                    LOGGER.error("Problem computing polygon area: " + e.getMessage());
                }
            }
            sfBuilder.set("Count", count);
            sfBuilder.set("Area", area);
            sfBuilder.set("Density", density);
            final SimpleFeature sf = sfBuilder.buildFeature("Hull-" + index);
            writer.write(sf);
        }
    }
    return featureAdapter;
}

Example 2 with SpatialOptions

use of org.locationtech.geowave.core.geotime.index.SpatialOptions in project geowave by locationtech.

the class KDERunner method run.

public void run() throws IOException {
    initContext();
    // Validate inputs
    if (inputDataStore == null) {
        LOGGER.error("You must supply an input datastore!");
        throw new IOException("You must supply an input datastore!");
    }
    // Retrieve the feature adapters
    final VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder();
    List<String> featureTypeNames;
    // If provided, just use the one
    if (typeName != null) {
        featureTypeNames = new ArrayList<>();
        featureTypeNames.add(typeName);
    } else {
        // otherwise, grab all the feature adapters
        featureTypeNames = FeatureDataUtils.getFeatureTypeNames(inputDataStore);
    }
    bldr.setTypeNames(featureTypeNames.toArray(new String[0]));
    if (indexName != null) {
        bldr.indexName(indexName);
    }
    Index inputPrimaryIndex = null;
    final Index[] idxArray = inputDataStore.createDataStore().getIndices();
    for (final Index idx : idxArray) {
        if ((idx != null) && ((indexName == null) || indexName.equals(idx.getName()))) {
            inputPrimaryIndex = idx;
            break;
        }
    }
    final CoordinateReferenceSystem inputIndexCrs = GeometryUtils.getIndexCrs(inputPrimaryIndex);
    final String inputCrsCode = GeometryUtils.getCrsCode(inputIndexCrs);
    Index outputPrimaryIndex = outputIndex;
    CoordinateReferenceSystem outputIndexCrs = null;
    final String outputCrsCode;
    if (outputPrimaryIndex != null) {
        outputIndexCrs = GeometryUtils.getIndexCrs(outputPrimaryIndex);
        outputCrsCode = GeometryUtils.getCrsCode(outputIndexCrs);
    } else {
        final SpatialDimensionalityTypeProvider sdp = new SpatialDimensionalityTypeProvider();
        final SpatialOptions so = sdp.createOptions();
        so.setCrs(inputCrsCode);
        outputPrimaryIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(so);
        outputIndexCrs = inputIndexCrs;
        outputCrsCode = inputCrsCode;
    }
    final CoordinateSystem cs = outputIndexCrs.getCoordinateSystem();
    final CoordinateSystemAxis csx = cs.getAxis(0);
    final CoordinateSystemAxis csy = cs.getAxis(1);
    final double xMax = csx.getMaximumValue();
    final double xMin = csx.getMinimumValue();
    final double yMax = csy.getMaximumValue();
    final double yMin = csy.getMinimumValue();
    if ((xMax == Double.POSITIVE_INFINITY) || (xMin == Double.NEGATIVE_INFINITY) || (yMax == Double.POSITIVE_INFINITY) || (yMin == Double.NEGATIVE_INFINITY)) {
        LOGGER.error("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
        throw new RuntimeException("Raster KDE resize with raster primary index CRS dimensions min/max equal to positive infinity or negative infinity is not supported");
    }
    if (cqlFilter != null) {
        bldr.constraints(bldr.constraintsFactory().cqlConstraints(cqlFilter));
    }
    // Load RDD from datastore
    final RDDOptions kdeOpts = new RDDOptions();
    kdeOpts.setMinSplits(minSplits);
    kdeOpts.setMaxSplits(maxSplits);
    kdeOpts.setQuery(bldr.build());
    final Function<Double, Double> identity = x -> x;
    final Function2<Double, Double, Double> sum = (final Double x, final Double y) -> {
        return x + y;
    };
    final RasterDataAdapter adapter = RasterUtils.createDataAdapterTypeDouble(coverageName, KDEReducer.NUM_BANDS, tileSize, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, new NoDataMergeStrategy());
    outputDataStore.createDataStore().addType(adapter, outputPrimaryIndex);
    // The following "inner" variables are created to give access to member
    // variables within lambda
    // expressions
    // tileSize;
    final int innerTileSize = 1;
    final String innerCoverageName = coverageName;
    for (int level = minLevel; level <= maxLevel; level++) {
        final int numXTiles = (int) Math.pow(2, level + 1);
        final int numYTiles = (int) Math.pow(2, level);
        // * tileSize;
        final int numXPosts = numXTiles;
        // * tileSize;
        final int numYPosts = numYTiles;
        final GeoWaveRDD kdeRDD = GeoWaveRDDLoader.loadRDD(session.sparkContext(), inputDataStore, kdeOpts);
        JavaPairRDD<Double, Long> cells = kdeRDD.getRawRDD().flatMapToPair(new GeoWaveCellMapper(numXPosts, numYPosts, xMin, xMax, yMin, yMax, inputCrsCode, outputCrsCode)).combineByKey(identity, sum, sum).mapToPair(item -> item.swap());
        cells = cells.partitionBy(new RangePartitioner(cells.getNumPartitions(), cells.rdd(), true, scala.math.Ordering.Double$.MODULE$, scala.reflect.ClassTag$.MODULE$.apply(Double.class))).sortByKey(false).cache();
        final long count = cells.count();
        if (count == 0) {
            LOGGER.warn("No cells produced by KDE");
            continue;
        }
        final double max = cells.first()._1;
        JavaRDD<GridCoverage> rdd = cells.zipWithIndex().map(t -> {
            final TileInfo tileInfo = fromCellIndexToTileInfo(t._1._2, numXPosts, numYPosts, numXTiles, numYTiles, xMin, xMax, yMin, yMax, innerTileSize);
            final WritableRaster raster = RasterUtils.createRasterTypeDouble(NUM_BANDS, innerTileSize);
            final double normalizedValue = t._1._1 / max;
            // because we are using a Double as the key, the ordering
            // isn't always completely reproducible as Double equals does not
            // take into account an epsilon
            final double percentile = (count - t._2) / ((double) count);
            raster.setSample(tileInfo.x, tileInfo.y, 0, t._1._1);
            raster.setSample(tileInfo.x, tileInfo.y, 1, normalizedValue);
            raster.setSample(tileInfo.x, tileInfo.y, 2, percentile);
            return RasterUtils.createCoverageTypeDouble(innerCoverageName, tileInfo.tileWestLon, tileInfo.tileEastLon, tileInfo.tileSouthLat, tileInfo.tileNorthLat, MINS_PER_BAND, MAXES_PER_BAND, NAME_PER_BAND, raster, GeometryUtils.DEFAULT_CRS_STR);
        });
        LOGGER.debug("Writing results to output store...");
        if (tileSize > 1) {
            // byte[] adapterBytes = PersistenceUtils.toBinary(adapter);
            // byte[] indexBytes = PersistenceUtils.toBinary(outputPrimaryIndex);
            rdd = rdd.flatMapToPair(new TransformTileSize(adapter, outputPrimaryIndex)).groupByKey().map(new MergeOverlappingTiles(adapter, outputPrimaryIndex));
        }
        RDDUtils.writeRasterToGeoWave(jsc.sc(), outputPrimaryIndex, outputDataStore, adapter, rdd);
        LOGGER.debug("Results successfully written!");
    }
}

Example 3 with SpatialOptions

use of org.locationtech.geowave.core.geotime.index.SpatialOptions in project geowave by locationtech.

the class KMeansDistortionMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KMeansDistortionMapReduce.KMeansDistortionMapper mapper = new KMeansDistortionMapReduce.KMeansDistortionMapper();
    final KMeansDistortionMapReduce.KMeansDistortionReduce reducer = new KMeansDistortionMapReduce.KMeansDistortionReduce();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KMeansDistortionMapReduce.class, CommonParameters.Common.DISTANCE_FUNCTION_CLASS), FeatureCentroidDistanceFn.class, DistanceFn.class);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    final PropertyManagement propManagement = new PropertyManagement();
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, ftype.getTypeName());
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, ftype.getName().getNamespaceURI());
    propManagement.store(GlobalParameters.Global.BATCH_ID, batchId);
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, SimpleFeatureCentroidExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, SimpleFeatureItemWrapperFactory.class);
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    NestedGroupCentroidAssignment.setParameters(mapDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
    serializations();
    capturedObjects.clear();
    final SimpleFeature feature = AnalyticFeature.createGeometryFeature(ftype, batchId, "123", "fred", grp1, 20.30203, factory.createPoint(new Coordinate(02.33, 0.23)), new String[] { "extra1" }, new double[] { 0.022 }, 1, 1, 0);
    propManagement.store(CentroidParameters.Centroid.ZOOM_LEVEL, 1);
    ingest(pluginOptions.createDataStore(), testObjectAdapter, index, feature);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
}

Example 4 with SpatialOptions

use of org.locationtech.geowave.core.geotime.index.SpatialOptions in project geowave by locationtech.

the class KSamplerMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KSamplerMapReduce.SampleMap<TestObject> mapper = new KSamplerMapReduce.SampleMap<>();
    final KSamplerMapReduce.SampleReducer<TestObject> reducer = new KSamplerMapReduce.SampleReducer<>();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    final DataTypeAdapter<?> adapter = AnalyticFeature.createGeometryFeatureAdapter("altoids", new String[] {}, "http://geowave.test.net", ClusteringUtils.CLUSTERING_CRS);
    final PropertyManagement propManagement = new PropertyManagement();
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, "altoids");
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, "http://geowave.test.net");
    propManagement.store(GlobalParameters.Global.BATCH_ID, "b1");
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, TestObjectExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, TestObjectItemWrapperFactory.class);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    CentroidManagerGeoWave.setParameters(mapDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    // TODO it seems the centroid adapter is required to have been written,
    // should this initialization be handled by the runner class rather than
    // externally such as in the test?
    final DataStore dataStore = store.getDataStoreOptions().createDataStore();
    final InternalAdapterStore internalAdapterStore = store.getDataStoreOptions().createInternalAdapterStore();
    dataStore.addType(adapter, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()));
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_RANK_FUNCTION), TestSamplingMidRankFunction.class, SamplingRankFunction.class);
    internalAdapterId = internalAdapterStore.getInitialAdapterId(testObjectAdapter.getTypeName());
    other = internalAdapterStore.getInitialAdapterId(adapter.getTypeName());
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), adapter.getTypeName(), other);
    mapDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), adapter.getTypeName(), other);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    reduceDriver.getConfiguration().set(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.DATA_TYPE_NAME), "altoids");
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.EXTRACTOR_CLASS), TestObjectExtractor.class, CentroidExtractor.class);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    serializations();
}

Example 5 with SpatialOptions

use of org.locationtech.geowave.core.geotime.index.SpatialOptions in project geowave by locationtech.

the class SqlResultsWriter method writeResults.

public void writeResults(String typeName) {
    if (typeName == null) {
        typeName = DEFAULT_TYPE_NAME;
        LOGGER.warn("Using default type name (adapter id): '" + DEFAULT_TYPE_NAME + "' for SQL output");
    }
    final StructType schema = results.schema();
    final SimpleFeatureType featureType = SchemaConverter.schemaToFeatureType(schema, typeName);
    final SimpleFeatureBuilder sfBuilder = new SimpleFeatureBuilder(featureType);
    final FeatureDataAdapter featureAdapter = new FeatureDataAdapter(featureType);
    final DataStore featureStore = outputDataStore.createDataStore();
    final Index featureIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    featureStore.addType(featureAdapter, featureIndex);
    try (Writer writer = featureStore.createWriter(featureAdapter.getTypeName())) {
        final List<Row> rows = results.collectAsList();
        for (int r = 0; r < rows.size(); r++) {
            final Row row = rows.get(r);
            for (int i = 0; i < schema.fields().length; i++) {
                final StructField field = schema.apply(i);
                final Object rowObj = row.apply(i);
                if (rowObj != null) {
                    if (field.name().equals("geom")) {
                        final Geometry geom = (Geometry) rowObj;
                        sfBuilder.set("geom", geom);
                    } else if (field.dataType() == DataTypes.TimestampType) {
                        final long millis = ((Timestamp) rowObj).getTime();
                        final Date date = new Date(millis);
                        sfBuilder.set(field.name(), date);
                    } else {
                        sfBuilder.set(field.name(), rowObj);
                    }
                }
            }
            final SimpleFeature sf = sfBuilder.buildFeature("result-" + nf.format(r));
            writer.write(sf);
        }
    }
}