Examples with DataStore org.locationtech.geowave.core.store.api.DataStore used on opensource projects

Example 1 with DataStore

use of org.locationtech.geowave.core.store.api.DataStore in project geowave by locationtech.

the class KMeansUtils method writeClusterHulls.

public static DataTypeAdapter writeClusterHulls(final JavaRDD<Vector> inputCentroids, final KMeansModel clusterModel, final DataStorePluginOptions outputDataStore, final String hullAdapterName, final boolean computeMetadata) {
    final JavaPairRDD<Integer, Iterable<Vector>> groupByRdd = KMeansHullGenerator.groupByIndex(inputCentroids, clusterModel);
    final JavaPairRDD<Integer, Geometry> hullRdd = KMeansHullGenerator.generateHullsRDD(groupByRdd);
    final SimpleFeatureTypeBuilder typeBuilder = new SimpleFeatureTypeBuilder();
    typeBuilder.setName(hullAdapterName);
    typeBuilder.setNamespaceURI(BasicFeatureTypes.DEFAULT_NAMESPACE);
    try {
        typeBuilder.setCRS(CRS.decode("EPSG:4326", true));
    } catch (final FactoryException e) {
        LOGGER.error(e.getMessage(), e);
    }
    final AttributeTypeBuilder attrBuilder = new AttributeTypeBuilder();
    typeBuilder.add(attrBuilder.binding(Geometry.class).nillable(false).buildDescriptor(Geometry.class.getName().toString()));
    typeBuilder.add(attrBuilder.binding(Integer.class).nillable(false).buildDescriptor("ClusterIndex"));
    typeBuilder.add(attrBuilder.binding(Integer.class).nillable(false).buildDescriptor("Count"));
    typeBuilder.add(attrBuilder.binding(Double.class).nillable(false).buildDescriptor("Area"));
    typeBuilder.add(attrBuilder.binding(Double.class).nillable(false).buildDescriptor("Density"));
    final SimpleFeatureType sfType = typeBuilder.buildFeatureType();
    final SimpleFeatureBuilder sfBuilder = new SimpleFeatureBuilder(sfType);
    final FeatureDataAdapter featureAdapter = new FeatureDataAdapter(sfType);
    final DataStore featureStore = outputDataStore.createDataStore();
    final Index featureIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final PolygonAreaCalculator polyCalc = (computeMetadata ? new PolygonAreaCalculator() : null);
    featureStore.addType(featureAdapter, featureIndex);
    try (Writer writer = featureStore.createWriter(featureAdapter.getTypeName())) {
        for (final Tuple2<Integer, Geometry> hull : hullRdd.collect()) {
            final Integer index = hull._1;
            final Geometry geom = hull._2;
            sfBuilder.set(Geometry.class.getName(), geom);
            sfBuilder.set("ClusterIndex", index);
            int count = 0;
            double area = 0.0;
            double density = 0.0;
            if (computeMetadata) {
                for (final Iterable<Vector> points : groupByRdd.lookup(index)) {
                    final Vector[] pointVec = Iterables.toArray(points, Vector.class);
                    count += pointVec.length;
                }
                try {
                    // HP Fortify "NULL Pointer Dereference" false positive
                    // Exception handling will catch if polyCalc is null
                    area = polyCalc.getAreaDensify(geom);
                    density = count / area;
                } catch (final Exception e) {
                    LOGGER.error("Problem computing polygon area: " + e.getMessage());
                }
            }
            sfBuilder.set("Count", count);
            sfBuilder.set("Area", area);
            sfBuilder.set("Density", density);
            final SimpleFeature sf = sfBuilder.buildFeature("Hull-" + index);
            writer.write(sf);
        }
    }
    return featureAdapter;
}

Example 2 with DataStore

use of org.locationtech.geowave.core.store.api.DataStore in project geowave by locationtech.

the class KSamplerMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KSamplerMapReduce.SampleMap<TestObject> mapper = new KSamplerMapReduce.SampleMap<>();
    final KSamplerMapReduce.SampleReducer<TestObject> reducer = new KSamplerMapReduce.SampleReducer<>();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    final DataTypeAdapter<?> adapter = AnalyticFeature.createGeometryFeatureAdapter("altoids", new String[] {}, "http://geowave.test.net", ClusteringUtils.CLUSTERING_CRS);
    final PropertyManagement propManagement = new PropertyManagement();
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, "altoids");
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, "http://geowave.test.net");
    propManagement.store(GlobalParameters.Global.BATCH_ID, "b1");
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, TestObjectExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, TestObjectItemWrapperFactory.class);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    CentroidManagerGeoWave.setParameters(mapDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    // TODO it seems the centroid adapter is required to have been written,
    // should this initialization be handled by the runner class rather than
    // externally such as in the test?
    final DataStore dataStore = store.getDataStoreOptions().createDataStore();
    final InternalAdapterStore internalAdapterStore = store.getDataStoreOptions().createInternalAdapterStore();
    dataStore.addType(adapter, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()));
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_RANK_FUNCTION), TestSamplingMidRankFunction.class, SamplingRankFunction.class);
    internalAdapterId = internalAdapterStore.getInitialAdapterId(testObjectAdapter.getTypeName());
    other = internalAdapterStore.getInitialAdapterId(adapter.getTypeName());
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), adapter.getTypeName(), other);
    mapDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), adapter.getTypeName(), other);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    reduceDriver.getConfiguration().set(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.DATA_TYPE_NAME), "altoids");
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.EXTRACTOR_CLASS), TestObjectExtractor.class, CentroidExtractor.class);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    serializations();
}

Example 3 with DataStore

use of org.locationtech.geowave.core.store.api.DataStore in project geowave by locationtech.

the class GWQLParserTest method testCasting.

@Test
public void testCasting() {
    final DataStore dataStore = createDataStore();
    String statement = "SELECT * FROM type WHERE pop::date BETWEEN '2020-01-01' AND '2020-01-02'";
    Statement gwStatement = GWQLParser.parseStatement(dataStore, statement);
    assertTrue(gwStatement instanceof SelectStatement);
    SelectStatement<?> selectStatement = (SelectStatement<?>) gwStatement;
    assertNotNull(selectStatement.getFilter());
    Filter filter = selectStatement.getFilter();
    assertTrue(filter instanceof TemporalBetween);
    final TemporalBetween between = (TemporalBetween) filter;
    assertTrue(between.getValue() instanceof TemporalFieldValue);
    assertEquals("pop", ((TemporalFieldValue) between.getValue()).getFieldName());
    assertTrue(between.getLowerBound() instanceof TemporalLiteral);
    assertTrue(between.getUpperBound() instanceof TemporalLiteral);
    statement = "SELECT * FROM type WHERE geometry = 'POINT(1 1)'::geometry";
    gwStatement = GWQLParser.parseStatement(dataStore, statement);
    assertTrue(gwStatement instanceof SelectStatement);
    selectStatement = (SelectStatement<?>) gwStatement;
    assertNotNull(selectStatement.getFilter());
    filter = selectStatement.getFilter();
    assertTrue(filter instanceof SpatialEqualTo);
    final SpatialEqualTo equals = (SpatialEqualTo) filter;
    assertTrue(equals.getExpression1() instanceof SpatialFieldValue);
    assertEquals("geometry", ((SpatialFieldValue) equals.getExpression1()).getFieldName());
    assertTrue(equals.getExpression2() instanceof SpatialLiteral);
}

Example 4 with DataStore

use of org.locationtech.geowave.core.store.api.DataStore in project geowave by locationtech.

the class GWQLParserTest method testTemporalPredicateFunctions.

@Test
public void testTemporalPredicateFunctions() {
    final DataStore dataStore = createDataStore();
    String statement = "SELECT * FROM type WHERE tcontains(start, '2020-01-01')";
    Statement gwStatement = GWQLParser.parseStatement(dataStore, statement);
    assertTrue(gwStatement instanceof SelectStatement);
    SelectStatement<?> selectStatement = (SelectStatement<?>) gwStatement;
    assertNotNull(selectStatement.getFilter());
    Filter filter = selectStatement.getFilter();
    // During is the inverse of contains, so the operands should be flipped
    assertTrue(filter instanceof During);
    BinaryTemporalPredicate predicate = (BinaryTemporalPredicate) filter;
    assertTrue(predicate.getExpression1() instanceof TemporalLiteral);
    assertEquals(TemporalExpression.stringToDate("2020-01-01").getTime(), ((TemporalLiteral) predicate.getExpression1()).getValue().getStart().toEpochMilli());
    assertEquals(TemporalExpression.stringToDate("2020-01-01").getTime(), ((TemporalLiteral) predicate.getExpression1()).getValue().getEnd().toEpochMilli());
    assertTrue(predicate.getExpression2() instanceof TemporalFieldValue);
    assertEquals("start", ((TemporalFieldValue) predicate.getExpression2()).getFieldName());
    statement = "SELECT * FROM type WHERE toverlaps(start, '2020-01-01')";
    gwStatement = GWQLParser.parseStatement(dataStore, statement);
    assertTrue(gwStatement instanceof SelectStatement);
    selectStatement = (SelectStatement<?>) gwStatement;
    assertNotNull(selectStatement.getFilter());
    filter = selectStatement.getFilter();
    assertTrue(filter instanceof TimeOverlaps);
    predicate = (BinaryTemporalPredicate) filter;
    assertTrue(predicate.getExpression1() instanceof TemporalFieldValue);
    assertEquals("start", ((TemporalFieldValue) predicate.getExpression1()).getFieldName());
    assertTrue(predicate.getExpression2() instanceof TemporalLiteral);
    assertEquals(TemporalExpression.stringToDate("2020-01-01").getTime(), ((TemporalLiteral) predicate.getExpression2()).getValue().getStart().toEpochMilli());
    assertEquals(TemporalExpression.stringToDate("2020-01-01").getTime(), ((TemporalLiteral) predicate.getExpression2()).getValue().getEnd().toEpochMilli());
}

Example 5 with DataStore

use of org.locationtech.geowave.core.store.api.DataStore in project geowave by locationtech.

the class RasterTileResizeSparkRunner method run.

public void run() throws IOException {
    initContext();
    // Validate inputs
    if (inputStoreOptions == null) {
        LOGGER.error("You must supply an input datastore!");
        throw new IOException("You must supply an input datastore!");
    }
    final InternalAdapterStore internalAdapterStore = inputStoreOptions.createInternalAdapterStore();
    final short internalAdapterId = internalAdapterStore.getAdapterId(rasterResizeOptions.getInputCoverageName());
    final DataTypeAdapter adapter = inputStoreOptions.createAdapterStore().getAdapter(internalAdapterId).getAdapter();
    if (adapter == null) {
        throw new IllegalArgumentException("Adapter for coverage '" + rasterResizeOptions.getInputCoverageName() + "' does not exist in namespace '" + inputStoreOptions.getGeoWaveNamespace() + "'");
    }
    Index index = null;
    final IndexStore indexStore = inputStoreOptions.createIndexStore();
    if (rasterResizeOptions.getIndexName() != null) {
        index = indexStore.getIndex(rasterResizeOptions.getIndexName());
    }
    if (index == null) {
        try (CloseableIterator<Index> indices = indexStore.getIndices()) {
            index = indices.next();
        }
        if (index == null) {
            throw new IllegalArgumentException("Index does not exist in namespace '" + inputStoreOptions.getGeoWaveNamespace() + "'");
        }
    }
    final RasterDataAdapter newAdapter = new RasterDataAdapter((RasterDataAdapter) adapter, rasterResizeOptions.getOutputCoverageName(), rasterResizeOptions.getOutputTileSize());
    final DataStore store = outputStoreOptions.createDataStore();
    store.addType(newAdapter, index);
    final short newInternalAdapterId = outputStoreOptions.createInternalAdapterStore().addTypeName(newAdapter.getTypeName());
    final RDDOptions options = new RDDOptions();
    if (rasterResizeOptions.getMinSplits() != null) {
        options.setMinSplits(rasterResizeOptions.getMinSplits());
    }
    if (rasterResizeOptions.getMaxSplits() != null) {
        options.setMaxSplits(rasterResizeOptions.getMaxSplits());
    }
    final JavaPairRDD<GeoWaveInputKey, GridCoverage> inputRDD = GeoWaveRDDLoader.loadRawRasterRDD(jsc.sc(), inputStoreOptions, index.getName(), rasterResizeOptions.getMinSplits(), rasterResizeOptions.getMaxSplits());
    LOGGER.debug("Writing results to output store...");
    RDDUtils.writeRasterToGeoWave(jsc.sc(), index, outputStoreOptions, newAdapter, inputRDD.flatMapToPair(new RasterResizeMappingFunction(internalAdapterId, newInternalAdapterId, newAdapter, index)).groupByKey().map(new MergeRasterFunction(internalAdapterId, newInternalAdapterId, newAdapter, index)));
    LOGGER.debug("Results successfully written!");
}