Examples with DataStorePluginOptions org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions used on opensource projects

Example 1 with DataStorePluginOptions

use of org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions in project geowave by locationtech.

the class RDDUtils method writeToGeoWave.

/**
 * Translate a set of objects in a JavaRDD to a provided type and push to GeoWave
 *
 * @throws IOException
 */
private static void writeToGeoWave(final SparkContext sc, final Index index, final DataStorePluginOptions outputStoreOptions, final DataTypeAdapter adapter, final JavaRDD<SimpleFeature> inputRDD) throws IOException {
    // setup the configuration and the output format
    final Configuration conf = new org.apache.hadoop.conf.Configuration(sc.hadoopConfiguration());
    GeoWaveOutputFormat.setStoreOptions(conf, outputStoreOptions);
    GeoWaveOutputFormat.addIndex(conf, index);
    GeoWaveOutputFormat.addDataAdapter(conf, adapter);
    // create the job
    final Job job = new Job(conf);
    job.setOutputKeyClass(GeoWaveOutputKey.class);
    job.setOutputValueClass(SimpleFeature.class);
    job.setOutputFormatClass(GeoWaveOutputFormat.class);
    // broadcast string names
    final ClassTag<String> stringTag = scala.reflect.ClassTag$.MODULE$.apply(String.class);
    final Broadcast<String> typeName = sc.broadcast(adapter.getTypeName(), stringTag);
    final Broadcast<String> indexName = sc.broadcast(index.getName(), stringTag);
    // map to a pair containing the output key and the output value
    inputRDD.mapToPair(feat -> new Tuple2<>(new GeoWaveOutputKey(typeName.value(), indexName.value()), feat)).saveAsNewAPIHadoopDataset(job.getConfiguration());
}

Example 2 with DataStorePluginOptions

use of org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions in project geowave by locationtech.

the class KMeansDistortionMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KMeansDistortionMapReduce.KMeansDistortionMapper mapper = new KMeansDistortionMapReduce.KMeansDistortionMapper();
    final KMeansDistortionMapReduce.KMeansDistortionReduce reducer = new KMeansDistortionMapReduce.KMeansDistortionReduce();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KMeansDistortionMapReduce.class, CommonParameters.Common.DISTANCE_FUNCTION_CLASS), FeatureCentroidDistanceFn.class, DistanceFn.class);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), adapterId);
    final PropertyManagement propManagement = new PropertyManagement();
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, ftype.getTypeName());
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, ftype.getName().getNamespaceURI());
    propManagement.store(GlobalParameters.Global.BATCH_ID, batchId);
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, SimpleFeatureCentroidExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, SimpleFeatureItemWrapperFactory.class);
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    NestedGroupCentroidAssignment.setParameters(mapDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
    serializations();
    capturedObjects.clear();
    final SimpleFeature feature = AnalyticFeature.createGeometryFeature(ftype, batchId, "123", "fred", grp1, 20.30203, factory.createPoint(new Coordinate(02.33, 0.23)), new String[] { "extra1" }, new double[] { 0.022 }, 1, 1, 0);
    propManagement.store(CentroidParameters.Centroid.ZOOM_LEVEL, 1);
    ingest(pluginOptions.createDataStore(), testObjectAdapter, index, feature);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KMeansDistortionMapReduce.class, propManagement);
}

Example 3 with DataStorePluginOptions

use of org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions in project geowave by locationtech.

the class KSamplerMapReduceTest method setUp.

@Before
public void setUp() throws IOException {
    final KSamplerMapReduce.SampleMap<TestObject> mapper = new KSamplerMapReduce.SampleMap<>();
    final KSamplerMapReduce.SampleReducer<TestObject> reducer = new KSamplerMapReduce.SampleReducer<>();
    mapDriver = MapDriver.newMapDriver(mapper);
    reduceDriver = ReduceDriver.newReduceDriver(reducer);
    final DataTypeAdapter<?> adapter = AnalyticFeature.createGeometryFeatureAdapter("altoids", new String[] {}, "http://geowave.test.net", ClusteringUtils.CLUSTERING_CRS);
    final PropertyManagement propManagement = new PropertyManagement();
    final DataStorePluginOptions pluginOptions = new DataStorePluginOptions();
    GeoWaveStoreFinder.getRegisteredStoreFactoryFamilies().put("memory", new MemoryStoreFactoryFamily());
    pluginOptions.selectPlugin("memory");
    final MemoryRequiredOptions opts = (MemoryRequiredOptions) pluginOptions.getFactoryOptions();
    final String namespace = "test_" + getClass().getName() + "_" + name.getMethodName();
    opts.setGeoWaveNamespace(namespace);
    final PersistableStore store = new PersistableStore(pluginOptions);
    propManagement.store(StoreParam.INPUT_STORE, store);
    propManagement.store(CentroidParameters.Centroid.INDEX_NAME, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()).getName());
    propManagement.store(CentroidParameters.Centroid.DATA_TYPE_ID, "altoids");
    propManagement.store(CentroidParameters.Centroid.DATA_NAMESPACE_URI, "http://geowave.test.net");
    propManagement.store(GlobalParameters.Global.BATCH_ID, "b1");
    propManagement.store(CentroidParameters.Centroid.EXTRACTOR_CLASS, TestObjectExtractor.class);
    propManagement.store(CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS, TestObjectItemWrapperFactory.class);
    CentroidManagerGeoWave.setParameters(reduceDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    CentroidManagerGeoWave.setParameters(mapDriver.getConfiguration(), KSamplerMapReduce.class, propManagement);
    // TODO it seems the centroid adapter is required to have been written,
    // should this initialization be handled by the runner class rather than
    // externally such as in the test?
    final DataStore dataStore = store.getDataStoreOptions().createDataStore();
    final InternalAdapterStore internalAdapterStore = store.getDataStoreOptions().createInternalAdapterStore();
    dataStore.addType(adapter, SpatialDimensionalityTypeProvider.createIndexFromOptions(new SpatialOptions()));
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_RANK_FUNCTION), TestSamplingMidRankFunction.class, SamplingRankFunction.class);
    internalAdapterId = internalAdapterStore.getInitialAdapterId(testObjectAdapter.getTypeName());
    other = internalAdapterStore.getInitialAdapterId(adapter.getTypeName());
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), testObjectAdapter);
    JobContextAdapterStore.addDataAdapter(mapDriver.getConfiguration(), adapter);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    JobContextInternalAdapterStore.addTypeName(mapDriver.getConfiguration(), adapter.getTypeName(), other);
    mapDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    reduceDriver.getConfiguration().setInt(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.SAMPLE_SIZE), 2);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), adapter);
    JobContextAdapterStore.addDataAdapter(reduceDriver.getConfiguration(), testObjectAdapter);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), adapter.getTypeName(), other);
    JobContextInternalAdapterStore.addTypeName(reduceDriver.getConfiguration(), testObjectAdapter.getTypeName(), internalAdapterId);
    reduceDriver.getConfiguration().set(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, SampleParameters.Sample.DATA_TYPE_NAME), "altoids");
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.EXTRACTOR_CLASS), TestObjectExtractor.class, CentroidExtractor.class);
    mapDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    reduceDriver.getConfiguration().setClass(GeoWaveConfiguratorBase.enumToConfKey(KSamplerMapReduce.class, CentroidParameters.Centroid.WRAPPER_FACTORY_CLASS), TestObjectItemWrapperFactory.class, AnalyticItemWrapperFactory.class);
    serializations();
}

Example 4 with DataStorePluginOptions

use of org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions in project geowave by locationtech.

the class SparkSqlCommand method initStores.

@SuppressFBWarnings("SF_SWITCH_FALLTHROUGH")
private String initStores(final File configFile, final String sql, final String outputStoreName, final Console console) {
    final Pattern storeDetect = Pattern.compile("(\\\"[^\\\"]*\\\"|'[^']*')|([%][^.,\\s]+)");
    final String escapedDelimRegex = java.util.regex.Pattern.quote(STORE_ADAPTER_DELIM);
    Matcher matchedStore = getFirstPositiveMatcher(storeDetect, sql);
    String replacedSQL = sql;
    while (matchedStore != null) {
        String parseStore = matchedStore.group(2);
        final String originalStoreText = parseStore;
        // Drop the first character off string should be % sign
        parseStore = parseStore.substring(1);
        parseStore = parseStore.trim();
        LOGGER.debug("parsed store: " + parseStore);
        final String[] storeNameParts = parseStore.split(escapedDelimRegex);
        LOGGER.debug("Split Count: " + storeNameParts.length);
        for (final String split : storeNameParts) {
            LOGGER.debug("Store split: " + split);
        }
        String storeName = null;
        String adapterName = null;
        String viewName = null;
        switch(storeNameParts.length) {
            case 3:
                viewName = storeNameParts[2].trim();
            case 2:
                adapterName = storeNameParts[1].trim();
            case 1:
                storeName = storeNameParts[0].trim();
                break;
            default:
                throw new ParameterException("Ambiguous datastore" + STORE_ADAPTER_DELIM + "adapter designation: " + Arrays.toString(storeNameParts));
        }
        // Attempt to load store.
        final DataStorePluginOptions storeOptions = CLIUtils.loadStore(storeName, configFile, console);
        viewName = sqlRunner.addInputStore(storeOptions, adapterName, viewName);
        if (viewName != null) {
            replacedSQL = StringUtils.replace(replacedSQL, originalStoreText, viewName, -1);
        }
        matchedStore = getNextPositiveMatcher(matchedStore);
    }
    return replacedSQL;
}

Example 5 with DataStorePluginOptions

use of org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions in project geowave by locationtech.

the class VectorIngestRunner method runInternal.

@Override
protected void runInternal(final OperationParams params) throws Exception {
    try {
        // Ensure we have all the required arguments
        if (parameters.size() != 2) {
            throw new ParameterException("Requires arguments: <store name> <comma delimited index list>");
        }
        final String providerName = sentinel2Options.providerName();
        final String inputStoreName = parameters.get(0);
        final String indexList = parameters.get(1);
        // Get the Sentinel2 provider.
        final Sentinel2ImageryProvider provider = Sentinel2ImageryProvider.getProvider(providerName);
        if (provider == null) {
            throw new RuntimeException("Unable to find '" + providerName + "' Sentinel2 provider");
        }
        // Config file
        final File configFile = (File) params.getContext().get(ConfigOptions.PROPERTIES_FILE_CONTEXT);
        final DataStorePluginOptions storeOptions = CLIUtils.loadStore(inputStoreName, configFile, params.getConsole());
        final DataStore store = storeOptions.createDataStore();
        // Load the Indices
        final Index[] indices = DataStoreUtils.loadIndices(storeOptions.createIndexStore(), indexList).toArray(new Index[0]);
        sceneType = provider.sceneFeatureTypeBuilder().buildFeatureType();
        final FeatureDataAdapter sceneAdapter = new FeatureDataAdapter(sceneType);
        store.addType(sceneAdapter, indices);
        sceneWriter = store.createWriter(sceneAdapter.getTypeName());
        final SimpleFeatureType bandType = provider.bandFeatureTypeBuilder().buildFeatureType();
        final FeatureDataAdapter bandAdapter = new FeatureDataAdapter(bandType);
        store.addType(bandAdapter, indices);
        bandWriter = store.createWriter(bandAdapter.getTypeName());
        super.runInternal(params);
    } finally {
        if (sceneWriter != null) {
            sceneWriter.close();
        }
        if (bandWriter != null) {
            bandWriter.close();
        }
    }
}