Examples with MemoryRequiredOptions org.locationtech.geowave.core.store.memory.MemoryRequiredOptions used on opensource projects

Example 1 with MemoryRequiredOptions

use of org.locationtech.geowave.core.store.memory.MemoryRequiredOptions in project geowave by locationtech.

the class KMeansDistortionMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KMeansDistortionMapReduce.KMeansDistortionMapper mapper = new KMeansDistortionMapReduce.KMeansDistortionMapper();
    final KMeansDistortionMapReduce.KMeansDistortionReduce reducer = new KMeansDistortionMapReduce.KMeansDistortionReduce();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KMeansDistortionMapReduce.class, CommonParameters.Common.DISTANCE_FUNCTION_CLASS), FeatureCentroidDistanceFn.class, DistanceFn.class);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    final PropertyManagement propManagement = new PropertyManagement();
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, ftype.getTypeName());
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, ftype.getName().getNamespaceURI());
    propManagement.store(GlobalParameters.Global.BATCH_ID, batchId);
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, SimpleFeatureCentroidExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, SimpleFeatureItemWrapperFactory.class);
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    NestedGroupCentroidAssignment.setParameters(mapDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
    serializations();
    capturedObjects.clear();
    final SimpleFeature feature = AnalyticFeature.createGeometryFeature(ftype, batchId, "123", "fred", grp1, 20.30203, factory.createPoint(new Coordinate(02.33, 0.23)), new String[] { "extra1" }, new double[] { 0.022 }, 1, 1, 0);
    propManagement.store(CentroidParameters.Centroid.ZOOM_LEVEL, 1);
    ingest(pluginOptions.createDataStore(), testObjectAdapter, index, feature);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
}

Example 2 with MemoryRequiredOptions

use of org.locationtech.geowave.core.store.memory.MemoryRequiredOptions in project geowave by locationtech.

the class KSamplerMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KSamplerMapReduce.SampleMap<TestObject> mapper = new KSamplerMapReduce.SampleMap<>();
    final KSamplerMapReduce.SampleReducer<TestObject> reducer = new KSamplerMapReduce.SampleReducer<>();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    final DataTypeAdapter<?> adapter = AnalyticFeature.createGeometryFeatureAdapter("altoids", new String[] {}, "http://geowave.test.net", ClusteringUtils.CLUSTERING_CRS);
    final PropertyManagement propManagement = new PropertyManagement();
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, "altoids");
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, "http://geowave.test.net");
    propManagement.store(GlobalParameters.Global.BATCH_ID, "b1");
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, TestObjectExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, TestObjectItemWrapperFactory.class);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    CentroidManagerGeoWave.setParameters(mapDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    // TODO it seems the centroid adapter is required to have been written,
    // should this initialization be handled by the runner class rather than
    // externally such as in the test?
    final DataStore dataStore = store.getDataStoreOptions().createDataStore();
    final InternalAdapterStore internalAdapterStore = store.getDataStoreOptions().createInternalAdapterStore();
    dataStore.addType(adapter, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()));
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_RANK_FUNCTION), TestSamplingMidRankFunction.class, SamplingRankFunction.class);
    internalAdapterId = internalAdapterStore.getInitialAdapterId(testObjectAdapter.getTypeName());
    other = internalAdapterStore.getInitialAdapterId(adapter.getTypeName());
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), adapter.getTypeName(), other);
    mapDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), adapter.getTypeName(), other);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    reduceDriver.getConfiguration().set(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.DATA_TYPE_NAME), "altoids");
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.EXTRACTOR_CLASS), TestObjectExtractor.class, CentroidExtractor.class);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    serializations();
}

Example 3 with MemoryRequiredOptions

use of org.locationtech.geowave.core.store.memory.MemoryRequiredOptions in project geowave by locationtech.

the class CustomStatisticExample method run.

public void run() {
    // Create an in-memory data store to use with this example
    dataStore = DataStoreFactory.createDataStore(new MemoryRequiredOptions());
    // Create the simple feature type for our data
    simpleFeatureType = getSimpleFeatureType();
    // Create an adapter for our features
    adapter = new FeatureDataAdapter(simpleFeatureType);
    // Create the spatial index
    spatialIndex = new SpatialIndexBuilder().createIndex();
    // Add the type to the data store with the spatial and custom indices
    dataStore.addType(adapter, spatialIndex);
    // Create a word count statistic on the `str` field of our type for all words
    final WordCountStatistic allWords = new WordCountStatistic();
    allWords.setTypeName(adapter.getTypeName());
    allWords.setFieldName("str");
    allWords.setMinWordLength(0);
    allWords.setTag("ALL_WORDS");
    // Create a word count statistic on the `str` field of our type for long words
    final WordCountStatistic longWords = new WordCountStatistic();
    longWords.setTypeName(adapter.getTypeName());
    longWords.setFieldName("str");
    longWords.setMinWordLength(5);
    longWords.setTag("LONG_WORDS");
    // Add the statistics
    dataStore.addStatistic(allWords, longWords);
    // Ingest the data into a spatial index
    ingestData();
    // Get the statistics
    System.out.println("Total number of words: " + dataStore.getStatisticValue(allWords));
    System.out.println("Total number of long words: " + dataStore.getStatisticValue(longWords));
    // You can also get the actual statistics from the data store at a later time
    final WordCountStatistic stat = (WordCountStatistic) dataStore.getFieldStatistic(WordCountStatistic.STATS_TYPE, adapter.getTypeName(), "str", "ALL_WORDS");
    System.out.println("ALL_WORDS Statistic: " + stat.toString());
}

Example 4 with MemoryRequiredOptions

use of org.locationtech.geowave.core.store.memory.MemoryRequiredOptions in project geowave by locationtech.

the class ConfigCacheIT method addStore.

@Test
public void addStore() {
    final String storeName = new MemoryStoreFactoryFamily().getType();
    final AddStoreCommand command = new AddStoreCommand();
    command.setParameters("abc");
    command.setMakeDefault(true);
    command.setStoreType(storeName);
    // This will load the params via SPI.
    command.prepare(operationParams);
    final DataStorePluginOptions options = command.getPluginOptions();
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) options.getFactoryOptions();
    opts.setGeoWaveNamespace("namespace");
    command.execute(operationParams);
    final Properties props = ConfigOptions.loadProperties(configFile);
    Assert.assertEquals("namespace", props.getProperty("store.abc.opts." + StoreFactoryOptions.GEOWAVE_NAMESPACE_OPTION));
    Assert.assertEquals("abc", props.getProperty(DataStorePluginOptions.DEFAULT_PROPERTY_NAMESPACE));
}

Example 5 with MemoryRequiredOptions

use of org.locationtech.geowave.core.store.memory.MemoryRequiredOptions in project geowave by locationtech.

the class SpatialBinningStatisticExample method main.

public static void main(final String[] args) {
    final SimpleFeatureType featureType = getSimpleFeatureType();
    // Points (to be ingested into GeoWave Data Store)
    final List<SimpleFeature> cannedFeatures = ImmutableList.of(buildSimpleFeature(featureType, "Loc1", new Coordinate(-77.0352, 38.8895), 12), buildSimpleFeature(featureType, "Loc2", new Coordinate(-77.0366, 38.8977), 13), buildSimpleFeature(featureType, "Loc3", new Coordinate(-76.8644, 38.9078), 8), buildSimpleFeature(featureType, "Loc4", new Coordinate(-76.350677, 38.9641511), 15), buildSimpleFeature(featureType, "Loc5", new Coordinate(-77.3384112, 38.416091), 7), buildSimpleFeature(featureType, "Loc6", new Coordinate(-67.0352, 28.8895), 3), buildSimpleFeature(featureType, "Loc7", new Coordinate(-67.0366, 28.8977), 99), buildSimpleFeature(featureType, "Loc8", new Coordinate(-66.8644, 28.9078), 0), buildSimpleFeature(featureType, "Loc9", new Coordinate(-66.350677, 28.9641511), 1), buildSimpleFeature(featureType, "Loc10", new Coordinate(-67.3384112, 28.416091), 23));
    final Index index = SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions());
    final DataStore dataStore = DataStoreFactory.createDataStore(new MemoryRequiredOptions());
    final FeatureDataAdapter adapter = new FeatureDataAdapter(featureType);
    final Envelope bbox1 = new Envelope(-77.5, -76, 38.4, 39);
    final Envelope bbox2 = new Envelope(-67.5, -66, 28.4, 29);
    dataStore.addType(adapter, index);
    final CountStatistic s2Count = new CountStatistic(featureType.getTypeName());
    s2Count.setTag("S2-Example");
    final SpatialFieldValueBinningStrategy s2SpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    // type could be Google's S2, Uber's H3, or simple GeoHash
    s2SpatialBinning.setType(SpatialBinningType.S2);
    // precision is the character length for H3 and GeoHash which is over twice as coarse as S2
    // which uses powers of two for precision (so a precision of 8 in S2 is actually a coarser
    // granularity than a precision of 4 in GeoHash or H3)
    s2SpatialBinning.setPrecision(7);
    s2Count.setBinningStrategy(s2SpatialBinning);
    final CountStatistic h3Count = new CountStatistic(featureType.getTypeName());
    // stats for the same feature type should have different tags
    h3Count.setTag("H3-Example");
    final SpatialFieldValueBinningStrategy h3SpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    // type could be Google's S2, Uber's H3, or simple GeoHash
    h3SpatialBinning.setType(SpatialBinningType.H3);
    h3SpatialBinning.setPrecision(3);
    h3Count.setBinningStrategy(h3SpatialBinning);
    final CountStatistic geohashCount = new CountStatistic(featureType.getTypeName());
    geohashCount.setTag("Geohash-Example");
    final SpatialFieldValueBinningStrategy geohashSpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    // type could be Google's S2, Uber's H3, or simple GeoHash
    geohashSpatialBinning.setType(SpatialBinningType.GEOHASH);
    geohashSpatialBinning.setPrecision(3);
    geohashCount.setBinningStrategy(geohashSpatialBinning);
    // you can add "empty" statistic before you've written any data, the stats will then be updated
    // as you write data
    // alternatively if you don't use the "empty" variant it will automatically calculate and update
    // these stats for pre-existing data before returning from the method
    dataStore.addEmptyStatistic(s2Count, h3Count, geohashCount);
    // Ingest cannedFeatures into the DataStore.
    try (Writer<SimpleFeature> indexWriter = dataStore.createWriter(adapter.getTypeName())) {
        for (final SimpleFeature sf : cannedFeatures) {
            indexWriter.write(sf);
        }
    }
    System.out.println("***** S2 Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(s2Count)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), s2SpatialBinning.binToString(pair.getLeft()), s2SpatialBinning.getType().getBinGeometry(pair.getLeft(), 7)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(s2Count, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), s2SpatialBinning.binToString(pair.getLeft()), s2SpatialBinning.getType().getBinGeometry(pair.getLeft(), 7)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** %d in bbox %s **", dataStore.getStatisticValue(s2Count, BinConstraints.ofObject(bbox2)), bbox2));
    System.out.println("\n***** H3 Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(h3Count)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), h3SpatialBinning.binToString(pair.getLeft()), h3SpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(h3Count, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), h3SpatialBinning.binToString(pair.getLeft()), h3SpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** %d in bbox %s **", dataStore.getStatisticValue(h3Count, BinConstraints.ofObject(bbox2)), bbox2));
    System.out.println("\n***** Geohash Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(geohashCount)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), geohashSpatialBinning.binToString(pair.getLeft()), geohashSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Long>> it = dataStore.getBinnedStatisticValues(geohashCount, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Long> pair = it.next();
            System.out.println(String.format("Count: %d, Bin: %s, Bin Geometry: %s", pair.getRight(), geohashSpatialBinning.binToString(pair.getLeft()), geohashSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** %d in bbox %s **", dataStore.getStatisticValue(geohashCount, BinConstraints.ofObject(bbox2)), bbox2));
    // and finally just to make it clear, you can apply spatial binning to *any* statistic not just
    // counts
    // so here's an example binning numeric stats of the population (sum, avg, std dev, etc.) by an
    // S2 level 7 grid
    final NumericStatsStatistic s2PopulationStats = new NumericStatsStatistic(featureType.getTypeName(), "population");
    s2PopulationStats.setTag("S2-Population-Stats");
    final SpatialFieldValueBinningStrategy s2PopulationSpatialBinning = new SpatialFieldValueBinningStrategy(featureType.getGeometryDescriptor().getLocalName());
    s2PopulationSpatialBinning.setType(SpatialBinningType.S2);
    s2PopulationSpatialBinning.setPrecision(7);
    s2PopulationStats.setBinningStrategy(s2PopulationSpatialBinning);
    // here we'll calculate the stat on add based on the already written data (rather than adding
    // the "empty" statistic)
    dataStore.addStatistic(s2PopulationStats);
    // and we'll run through the same set of examples of getting all the bins and then filtering by
    // an envelope
    System.out.println("\n***** S2 Population Stats Binning *****");
    System.out.println("** All Bins **");
    try (CloseableIterator<Pair<ByteArray, Stats>> it = dataStore.getBinnedStatisticValues(s2PopulationStats)) {
        // you can get all bins
        while (it.hasNext()) {
            final Pair<ByteArray, Stats> pair = it.next();
            System.out.println(String.format("Population: %s, Bin: %s, Bin Geometry: %s", pair.getRight(), s2PopulationSpatialBinning.binToString(pair.getLeft()), s2PopulationSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    System.out.println(String.format("** Bins Within Envelope %s **", bbox1));
    try (CloseableIterator<Pair<ByteArray, Stats>> it = dataStore.getBinnedStatisticValues(s2PopulationStats, BinConstraints.ofObject(bbox1))) {
        // or you can get only bins within an envelope
        while (it.hasNext()) {
            final Pair<ByteArray, Stats> pair = it.next();
            System.out.println(String.format("Population: %s, Bin: %s, Bin Geometry: %s", pair.getRight(), s2PopulationSpatialBinning.binToString(pair.getLeft()), s2PopulationSpatialBinning.getType().getBinGeometry(pair.getLeft(), 3)));
        }
    }
    // or you could just get the aggregated statistic value for an envelope (keep in mind this is
    // using the statistic bins that intersect the envelope so may be an over-estimate for bins that
    // only partially intersect)
    System.out.println(String.format("** Population Stats '%s' in bbox %s **", dataStore.getStatisticValue(s2PopulationStats, BinConstraints.ofObject(bbox2)), bbox2));
}