Examples with PersistableStore org.locationtech.geowave.analytic.store.PersistableStore used on opensource projects

Example 1 with PersistableStore

use of org.locationtech.geowave.analytic.store.PersistableStore in project geowave by locationtech.

the class KMeansDistortionMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KMeansDistortionMapReduce.KMeansDistortionMapper mapper = new KMeansDistortionMapReduce.KMeansDistortionMapper();
    final KMeansDistortionMapReduce.KMeansDistortionReduce reducer = new KMeansDistortionMapReduce.KMeansDistortionReduce();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KMeansDistortionMapReduce.class, CommonParameters.Common.DISTANCE_FUNCTION_CLASS), FeatureCentroidDistanceFn.class, DistanceFn.class);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    final PropertyManagement propManagement = new PropertyManagement();
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, ftype.getTypeName());
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, ftype.getName().getNamespaceURI());
    propManagement.store(GlobalParameters.Global.BATCH_ID, batchId);
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, SimpleFeatureCentroidExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, SimpleFeatureItemWrapperFactory.class);
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    NestedGroupCentroidAssignment.setParameters(mapDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
    serializations();
    capturedObjects.clear();
    final SimpleFeature feature = AnalyticFeature.createGeometryFeature(ftype, batchId, "123", "fred", grp1, 20.30203, factory.createPoint(new Coordinate(02.33, 0.23)), new String[] { "extra1" }, new double[] { 0.022 }, 1, 1, 0);
    propManagement.store(CentroidParameters.Centroid.ZOOM_LEVEL, 1);
    ingest(pluginOptions.createDataStore(), testObjectAdapter, index, feature);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
}

Example 2 with PersistableStore

use of org.locationtech.geowave.analytic.store.PersistableStore in project geowave by locationtech.

the class KSamplerMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KSamplerMapReduce.SampleMap<TestObject> mapper = new KSamplerMapReduce.SampleMap<>();
    final KSamplerMapReduce.SampleReducer<TestObject> reducer = new KSamplerMapReduce.SampleReducer<>();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    final DataTypeAdapter<?> adapter = AnalyticFeature.createGeometryFeatureAdapter("altoids", new String[] {}, "http://geowave.test.net", ClusteringUtils.CLUSTERING_CRS);
    final PropertyManagement propManagement = new PropertyManagement();
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, "altoids");
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, "http://geowave.test.net");
    propManagement.store(GlobalParameters.Global.BATCH_ID, "b1");
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, TestObjectExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, TestObjectItemWrapperFactory.class);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    CentroidManagerGeoWave.setParameters(mapDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    // TODO it seems the centroid adapter is required to have been written,
    // should this initialization be handled by the runner class rather than
    // externally such as in the test?
    final DataStore dataStore = store.getDataStoreOptions().createDataStore();
    final InternalAdapterStore internalAdapterStore = store.getDataStoreOptions().createInternalAdapterStore();
    dataStore.addType(adapter, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()));
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_RANK_FUNCTION), TestSamplingMidRankFunction.class, SamplingRankFunction.class);
    internalAdapterId = internalAdapterStore.getInitialAdapterId(testObjectAdapter.getTypeName());
    other = internalAdapterStore.getInitialAdapterId(adapter.getTypeName());
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), adapter.getTypeName(), other);
    mapDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), adapter.getTypeName(), other);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    reduceDriver.getConfiguration().set(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.DATA_TYPE_NAME), "altoids");
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.EXTRACTOR_CLASS), TestObjectExtractor.class, CentroidExtractor.class);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    serializations();
}

Example 3 with PersistableStore

use of org.locationtech.geowave.analytic.store.PersistableStore in project geowave by locationtech.

the class KmeansParallelCommand method execute.

@Override
public void execute(final OperationParams params) throws Exception {
    // Ensure we have all the required arguments
    if (parameters.size() != 1) {
        throw new ParameterException("Requires arguments: <storename>");
    }
    final String inputStoreName = parameters.get(0);
    // Config file
    final File configFile = getGeoWaveConfigFile(params);
    if (commonOptions.getMapReduceHdfsHostPort() == null) {
        final Properties configProperties = ConfigOptions.loadProperties(configFile);
        final String hdfsFSUrl = ConfigHDFSCommand.getHdfsUrl(configProperties);
        commonOptions.setMapReduceHdfsHostPort(hdfsFSUrl);
    }
    // Attempt to load store.
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, configFile, params.getConsole());
    // Save a reference to the store in the property management.
    final PersistableStore persistedStore = new PersistableStore(inputStoreOptions);
    final PropertyManagement properties = new PropertyManagement();
    properties.store(StoreParameters.StoreParam.INPUT_STORE, persistedStore);
    // Convert properties from DBScanOptions and CommonOptions
    final PropertyManagementConverter converter = new PropertyManagementConverter(properties);
    converter.readProperties(commonOptions);
    converter.readProperties(kmeansCommonOptions);
    converter.readProperties(kmeansParallelOptions);
    properties.store(Extract.QUERY, commonOptions.buildQuery());
    final MultiLevelKMeansClusteringJobRunner runner = new MultiLevelKMeansClusteringJobRunner();
    final int status = runner.run(properties);
    if (status != 0) {
        throw new RuntimeException("Failed to execute: " + status);
    }
}

Example 4 with PersistableStore

use of org.locationtech.geowave.analytic.store.PersistableStore in project geowave by locationtech.

the class NearestNeighborCommand method computeResults.

@Override
public Void computeResults(final OperationParams params) throws Exception {
    // Ensure we have all the required arguments
    if ((parameters.size() != 1) && (inputStoreOptions == null)) {
        throw new ParameterException("Requires arguments: <storename>");
    }
    final String inputStoreName = parameters.get(0);
    // Config file
    final File configFile = getGeoWaveConfigFile(params);
    if (commonOptions.getMapReduceHdfsHostPort() == null) {
        final Properties configProperties = ConfigOptions.loadProperties(configFile);
        final String hdfsFSUrl = ConfigHDFSCommand.getHdfsUrl(configProperties);
        commonOptions.setMapReduceHdfsHostPort(hdfsFSUrl);
    }
    // Attempt to load store.
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, getGeoWaveConfigFile(params), params.getConsole());
    // Save a reference to the store in the property management.
    final PersistableStore persistedStore = new PersistableStore(inputStoreOptions);
    final PropertyManagement properties = new PropertyManagement();
    properties.store(StoreParameters.StoreParam.INPUT_STORE, persistedStore);
    // Convert properties from DBScanOptions and CommonOptions
    final PropertyManagementConverter converter = new PropertyManagementConverter(properties);
    converter.readProperties(commonOptions);
    converter.readProperties(nnOptions);
    properties.store(Extract.QUERY, commonOptions.buildQuery());
    final GeoWaveExtractNNJobRunner runner = new GeoWaveExtractNNJobRunner();
    final int status = runner.run(properties);
    if (status != 0) {
        throw new RuntimeException("Failed to execute: " + status);
    }
    return null;
}

Example 5 with PersistableStore

use of org.locationtech.geowave.analytic.store.PersistableStore in project geowave by locationtech.

the class DBScanCommand method computeResults.

@Override
public Void computeResults(final OperationParams params) throws Exception {
    final String inputStoreName = parameters.get(0);
    // Config file
    final File configFile = getGeoWaveConfigFile(params);
    if (commonOptions.getMapReduceHdfsHostPort() == null) {
        final Properties configProperties = ConfigOptions.loadProperties(configFile);
        final String hdfsFSUrl = ConfigHDFSCommand.getHdfsUrl(configProperties);
        commonOptions.setMapReduceHdfsHostPort(hdfsFSUrl);
    }
    // Attempt to load store.
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, configFile, params.getConsole());
    // Save a reference to the store in the property management.
    final PersistableStore persistedStore = new PersistableStore(inputStoreOptions);
    final PropertyManagement properties = new PropertyManagement();
    properties.store(StoreParameters.StoreParam.INPUT_STORE, persistedStore);
    // Convert properties from DBScanOptions and CommonOptions
    final PropertyManagementConverter converter = new PropertyManagementConverter(properties);
    converter.readProperties(commonOptions);
    converter.readProperties(dbScanOptions);
    properties.store(Extract.QUERY, commonOptions.buildQuery());
    final DBScanIterationsJobRunner runner = new DBScanIterationsJobRunner();
    final int status = runner.run(properties);
    if (status != 0) {
        throw new RuntimeException("Failed to execute: " + status);
    }
    return null;
}