Examples with LocalFileIngestCLIDriver org.locationtech.geowave.core.ingest.local.LocalFileIngestCLIDriver used on opensource projects

Example 1 with LocalFileIngestCLIDriver

use of org.locationtech.geowave.core.ingest.local.LocalFileIngestCLIDriver in project geowave by locationtech.

the class SparkIngestDriver method processInput.

public void processInput(final File configFile, final LocalInputCommandLineOptions localInput, final String inputStoreName, final String indexList, final VisibilityOptions visibilityOptions, final Properties configProperties, final Iterator<URI> inputFiles, final Console console) throws IOException {
    // Based on the selected formats, select the format plugins
    final IngestFormatPluginOptions pluginFormats = new IngestFormatPluginOptions();
    // Based on the selected formats, select the format plugins
    pluginFormats.selectPlugin(localInput.getFormats());
    DataStorePluginOptions inputStoreOptions = null;
    List<Index> indices = null;
    // Ingest Plugins
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = pluginFormats.createLocalIngestPlugins();
    inputStoreOptions = CLIUtils.loadStore(configProperties, inputStoreName, configFile, console);
    final IndexStore indexStore = inputStoreOptions.createIndexStore();
    indices = DataStoreUtils.loadIndices(indexStore, indexList);
    // first collect the local file ingest plugins
    final Map<String, LocalFileIngestPlugin<?>> localFileIngestPlugins = new HashMap<>();
    final List<DataTypeAdapter<?>> adapters = new ArrayList<>();
    for (final Entry<String, LocalFileIngestPlugin<?>> pluginEntry : ingestPlugins.entrySet()) {
        if (!IngestUtils.checkIndexesAgainstProvider(pluginEntry.getKey(), pluginEntry.getValue(), indices)) {
            continue;
        }
        localFileIngestPlugins.put(pluginEntry.getKey(), pluginEntry.getValue());
        adapters.addAll(Arrays.asList(pluginEntry.getValue().getDataAdapters()));
    }
    final LocalFileIngestCLIDriver localIngestDriver = new LocalFileIngestCLIDriver(inputStoreOptions, indices, localFileIngestPlugins, visibilityOptions, localInput, 1);
    localIngestDriver.startExecutor();
    final DataStore dataStore = inputStoreOptions.createDataStore();
    try (LocalIngestRunData runData = new LocalIngestRunData(adapters, dataStore, visibilityOptions.getConfiguredVisibilityHandler())) {
        final List<PluginVisitor<LocalFileIngestPlugin<?>>> pluginVisitors = new ArrayList<>(localFileIngestPlugins.size());
        for (final Entry<String, LocalFileIngestPlugin<?>> localPlugin : localFileIngestPlugins.entrySet()) {
            pluginVisitors.add(new PluginVisitor<LocalFileIngestPlugin<?>>(localPlugin.getValue(), localPlugin.getKey(), localInput.getExtensions()));
        }
        while (inputFiles.hasNext()) {
            final URL file = inputFiles.next().toURL();
            for (final PluginVisitor<LocalFileIngestPlugin<?>> visitor : pluginVisitors) {
                if (visitor.supportsFile(file)) {
                    localIngestDriver.processFile(file, visitor.getTypeName(), visitor.getLocalPluginBase(), runData);
                }
            }
        }
    } catch (final MalformedURLException e) {
        LOGGER.error("Error in converting input path to URL for " + inputFiles, e);
        throw new MalformedURLException("Error in converting input path to URL for " + inputFiles);
    } catch (final Exception e) {
        LOGGER.error("Error processing in processing input", e);
        throw new RuntimeException("Error processing in processing input", e);
    } finally {
        localIngestDriver.shutdownExecutor();
    }
}

Example 2 with LocalFileIngestCLIDriver

use of org.locationtech.geowave.core.ingest.local.LocalFileIngestCLIDriver in project geowave by locationtech.

the class LocalToGeoWaveCommand method computeResults.

@Override
public Void computeResults(final OperationParams params) {
    // Ensure we have all the required arguments
    if (parameters.size() != 3) {
        throw new ParameterException("Requires arguments: <file or directory> <storename> <comma delimited index list>");
    }
    final String inputPath = parameters.get(0);
    final String inputStoreName = parameters.get(1);
    final String indexList = parameters.get(2);
    // Config file
    final File configFile = getGeoWaveConfigFile(params);
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, configFile, params.getConsole());
    final IndexStore indexStore = inputStoreOptions.createIndexStore();
    inputIndices = DataStoreUtils.loadIndices(indexStore, indexList);
    // Ingest Plugins
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = pluginFormats.createLocalIngestPlugins();
    // Driver
    final LocalFileIngestCLIDriver driver = new LocalFileIngestCLIDriver(inputStoreOptions, inputIndices, ingestPlugins, visibilityOptions, localInputOptions, threads);
    // Execute
    if (!driver.runOperation(inputPath, configFile)) {
        throw new RuntimeException("Ingest failed to execute");
    }
    return null;
}