Examples with IngestFormatPluginOptions org.locationtech.geowave.core.ingest.operations.options.IngestFormatPluginOptions used on opensource projects

Example 1 with IngestFormatPluginOptions

use of org.locationtech.geowave.core.ingest.operations.options.IngestFormatPluginOptions in project geowave by locationtech.

the class KafkaTestUtils method testKafkaIngest.

public static void testKafkaIngest(final DataStorePluginOptions options, final boolean spatialTemporal, final String ingestFilePath) throws Exception {
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    // // FIXME
    // final String[] args = StringUtils.split("-kafkaingest" +
    // " -f gpx -batchSize 1 -consumerTimeoutMs 5000 -reconnectOnTimeout
    // -groupId testGroup"
    // + " -autoOffsetReset smallest -fetchMessageMaxBytes " +
    // MAX_MESSAGE_BYTES +
    // " -zookeeperConnect " + zookeeper + " -" +
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin("gpx");
    // Indexes
    final IndexPluginOptions indexOption = new IndexPluginOptions();
    indexOption.selectPlugin((spatialTemporal ? "spatial_temporal" : "spatial"));
    // Execute Command
    final KafkaToGeoWaveCommand kafkaToGeowave = new KafkaToGeoWaveCommand();
    final File configFile = File.createTempFile("test_stats", null);
    final ManualOperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(options);
    addStore.execute(params);
    final IndexStore indexStore = options.createIndexStore();
    final DataStore dataStore = options.createDataStore();
    if (indexStore.getIndex("testIndex") == null) {
        indexOption.setName("testIndex");
        dataStore.addIndex(indexOption.createIndex(dataStore));
    }
    kafkaToGeowave.setPluginFormats(ingestFormatOptions);
    kafkaToGeowave.getKafkaOptions().setBootstrapServers(KafkaTestEnvironment.getInstance().getBootstrapServers());
    kafkaToGeowave.getKafkaOptions().setConsumerTimeoutMs("5000");
    kafkaToGeowave.getKafkaOptions().setReconnectOnTimeout(false);
    kafkaToGeowave.getKafkaOptions().setGroupId("testGroup");
    kafkaToGeowave.getKafkaOptions().setAutoOffsetReset("earliest");
    kafkaToGeowave.getKafkaOptions().setMaxPartitionFetchBytes(MAX_MESSAGE_BYTES);
    kafkaToGeowave.setParameters("test-store", "testIndex");
    kafkaToGeowave.execute(params);
    // finish.
    try {
        kafkaToGeowave.getDriver().waitFutures();
    } catch (InterruptedException | ExecutionException e) {
        throw new RuntimeException(e);
    }
}

Example 2 with IngestFormatPluginOptions

use of org.locationtech.geowave.core.ingest.operations.options.IngestFormatPluginOptions in project geowave by locationtech.

the class MapReduceTestUtils method testMapReduceIngest.

public static void testMapReduceIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String format, final String ingestFilePath) throws Exception {
    // ingest gpx data directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    LOGGER.warn("Ingesting '" + ingestFilePath + "' - this may take several minutes...");
    final Thread progressLogger = startProgressLogger();
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        indexOptions.add(indexOption);
    }
    // Ingest Formats
    final MapReduceTestEnvironment env = MapReduceTestEnvironment.getInstance();
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // create temporary config file and use it for hdfs FS URL config
    final File configFile = File.createTempFile("test_mr", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final ConfigHDFSCommand configHdfs = new ConfigHDFSCommand();
    configHdfs.setHdfsUrlParameter(env.getHdfs());
    configHdfs.execute(operationParams);
    final LocalToMapReduceToGeoWaveCommand mrGw = new LocalToMapReduceToGeoWaveCommand();
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final DataStore geowaveDataStore = dataStore.createDataStore();
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        String indexName = "testIndex" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    mrGw.setPluginFormats(ingestFormatOptions);
    mrGw.setParameters(ingestFilePath, env.getHdfsBaseDirectory(), "test-store", indexParam.toString());
    mrGw.getMapReduceOptions().setJobTrackerHostPort(env.getJobtracker());
    mrGw.execute(operationParams);
    progressLogger.interrupt();
}

Example 3 with IngestFormatPluginOptions

use of org.locationtech.geowave.core.ingest.operations.options.IngestFormatPluginOptions in project geowave by locationtech.

the class TestUtils method testLocalIngest.

public static void testLocalIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String crsCode, final String ingestFilePath, final String format, final int nthreads, final boolean supportTimeRange) throws Exception {
    // ingest a shapefile (geotools type) directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        if (crsCode != null) {
            if (indexOption.getDimensionalityOptions() instanceof SpatialOptions) {
                ((SpatialOptions) indexOption.getDimensionalityOptions()).setCrs(crsCode);
            } else if (indexOption.getDimensionalityOptions() instanceof SpatialTemporalOptions) {
                ((SpatialTemporalOptions) indexOption.getDimensionalityOptions()).setCrs(crsCode);
            }
        }
        if (indexOption.getDimensionalityOptions() instanceof TemporalOptions) {
            ((TemporalOptions) indexOption.getDimensionalityOptions()).setNoTimeRanges(!supportTimeRange);
        }
        indexOptions.add(indexOption);
    }
    final File configFile = File.createTempFile("test_stats", null);
    final ManualOperationParams params = new ManualOperationParams();
    params.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    // Add Store
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(params);
    final IndexStore indexStore = dataStore.createIndexStore();
    final org.locationtech.geowave.core.store.api.DataStore geowaveDataStore = dataStore.createDataStore();
    // Add indices
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        final String indexName = "test-index" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    // Create the command and execute.
    final LocalToGeoWaveCommand localIngester = new LocalToGeoWaveCommand();
    localIngester.setPluginFormats(ingestFormatOptions);
    localIngester.setParameters(ingestFilePath, "test-store", indexParam.toString());
    localIngester.setThreads(nthreads);
    localIngester.execute(params);
    verifyStats(dataStore);
}

Example 4 with IngestFormatPluginOptions

use of org.locationtech.geowave.core.ingest.operations.options.IngestFormatPluginOptions in project geowave by locationtech.

the class TestUtils method testS3LocalIngest.

public static void testS3LocalIngest(final DataStorePluginOptions dataStore, final DimensionalityType dimensionalityType, final String s3Url, final String ingestFilePath, final String format, final int nthreads) throws Exception {
    // ingest a shapefile (geotools type) directly into GeoWave using the
    // ingest framework's main method and pre-defined commandline arguments
    // Ingest Formats
    final IngestFormatPluginOptions ingestFormatOptions = new IngestFormatPluginOptions();
    ingestFormatOptions.selectPlugin(format);
    // Indexes
    final String[] indexTypes = dimensionalityType.getDimensionalityArg().split(",");
    final List<IndexPluginOptions> indexOptions = new ArrayList<>(indexTypes.length);
    for (final String indexType : indexTypes) {
        final IndexPluginOptions indexOption = new IndexPluginOptions();
        indexOption.selectPlugin(indexType);
        indexOptions.add(indexOption);
    }
    final File configFile = File.createTempFile("test_s3_local_ingest", null);
    final ManualOperationParams operationParams = new ManualOperationParams();
    operationParams.getContext().put(ConfigOptions.PROPERTIES_FILE_CONTEXT, configFile);
    final AddStoreCommand addStore = new AddStoreCommand();
    addStore.setParameters("test-store");
    addStore.setPluginOptions(dataStore);
    addStore.execute(operationParams);
    final IndexStore indexStore = dataStore.createIndexStore();
    final org.locationtech.geowave.core.store.api.DataStore geowaveDataStore = dataStore.createDataStore();
    final StringBuilder indexParam = new StringBuilder();
    for (int i = 0; i < indexOptions.size(); i++) {
        final String indexName = "test-index" + i;
        if (indexStore.getIndex(indexName) == null) {
            indexOptions.get(i).setName(indexName);
            geowaveDataStore.addIndex(indexOptions.get(i).createIndex(geowaveDataStore));
        }
        indexParam.append(indexName + ",");
    }
    final ConfigAWSCommand configS3 = new ConfigAWSCommand();
    configS3.setS3UrlParameter(s3Url);
    configS3.execute(operationParams);
    // Create the command and execute.
    final LocalToGeoWaveCommand localIngester = new LocalToGeoWaveCommand();
    localIngester.setPluginFormats(ingestFormatOptions);
    localIngester.setParameters(ingestFilePath, "test-store", indexParam.toString());
    localIngester.setThreads(nthreads);
    localIngester.execute(operationParams);
    verifyStats(dataStore);
}

Example 5 with IngestFormatPluginOptions

use of org.locationtech.geowave.core.ingest.operations.options.IngestFormatPluginOptions in project geowave by locationtech.

the class SparkIngestDriver method processInput.

public void processInput(final File configFile, final LocalInputCommandLineOptions localInput, final String inputStoreName, final String indexList, final VisibilityOptions visibilityOptions, final Properties configProperties, final Iterator<URI> inputFiles, final Console console) throws IOException {
    // Based on the selected formats, select the format plugins
    final IngestFormatPluginOptions pluginFormats = new IngestFormatPluginOptions();
    // Based on the selected formats, select the format plugins
    pluginFormats.selectPlugin(localInput.getFormats());
    DataStorePluginOptions inputStoreOptions = null;
    List<Index> indices = null;
    // Ingest Plugins
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = pluginFormats.createLocalIngestPlugins();
    inputStoreOptions = CLIUtils.loadStore(configProperties, inputStoreName, configFile, console);
    final IndexStore indexStore = inputStoreOptions.createIndexStore();
    indices = DataStoreUtils.loadIndices(indexStore, indexList);
    // first collect the local file ingest plugins
    final Map<String, LocalFileIngestPlugin<?>> localFileIngestPlugins = new HashMap<>();
    final List<DataTypeAdapter<?>> adapters = new ArrayList<>();
    for (final Entry<String, LocalFileIngestPlugin<?>> pluginEntry : ingestPlugins.entrySet()) {
        if (!IngestUtils.checkIndexesAgainstProvider(pluginEntry.getKey(), pluginEntry.getValue(), indices)) {
            continue;
        }
        localFileIngestPlugins.put(pluginEntry.getKey(), pluginEntry.getValue());
        adapters.addAll(Arrays.asList(pluginEntry.getValue().getDataAdapters()));
    }
    final LocalFileIngestCLIDriver localIngestDriver = new LocalFileIngestCLIDriver(inputStoreOptions, indices, localFileIngestPlugins, visibilityOptions, localInput, 1);
    localIngestDriver.startExecutor();
    final DataStore dataStore = inputStoreOptions.createDataStore();
    try (LocalIngestRunData runData = new LocalIngestRunData(adapters, dataStore, visibilityOptions.getConfiguredVisibilityHandler())) {
        final List<PluginVisitor<LocalFileIngestPlugin<?>>> pluginVisitors = new ArrayList<>(localFileIngestPlugins.size());
        for (final Entry<String, LocalFileIngestPlugin<?>> localPlugin : localFileIngestPlugins.entrySet()) {
            pluginVisitors.add(new PluginVisitor<LocalFileIngestPlugin<?>>(localPlugin.getValue(), localPlugin.getKey(), localInput.getExtensions()));
        }
        while (inputFiles.hasNext()) {
            final URL file = inputFiles.next().toURL();
            for (final PluginVisitor<LocalFileIngestPlugin<?>> visitor : pluginVisitors) {
                if (visitor.supportsFile(file)) {
                    localIngestDriver.processFile(file, visitor.getTypeName(), visitor.getLocalPluginBase(), runData);
                }
            }
        }
    } catch (final MalformedURLException e) {
        LOGGER.error("Error in converting input path to URL for " + inputFiles, e);
        throw new MalformedURLException("Error in converting input path to URL for " + inputFiles);
    } catch (final Exception e) {
        LOGGER.error("Error processing in processing input", e);
        throw new RuntimeException("Error processing in processing input", e);
    } finally {
        localIngestDriver.shutdownExecutor();
    }
}