Examples with LocalIngestRunData org.locationtech.geowave.core.store.ingest.LocalIngestRunData used on opensource projects

Example 1 with LocalIngestRunData

use of org.locationtech.geowave.core.store.ingest.LocalIngestRunData in project geowave by locationtech.

the class SparkIngestDriver method processInput.

public void processInput(final File configFile, final LocalInputCommandLineOptions localInput, final String inputStoreName, final String indexList, final VisibilityOptions visibilityOptions, final Properties configProperties, final Iterator<URI> inputFiles, final Console console) throws IOException {
    // Based on the selected formats, select the format plugins
    final IngestFormatPluginOptions pluginFormats = new IngestFormatPluginOptions();
    // Based on the selected formats, select the format plugins
    pluginFormats.selectPlugin(localInput.getFormats());
    DataStorePluginOptions inputStoreOptions = null;
    List<Index> indices = null;
    // Ingest Plugins
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = pluginFormats.createLocalIngestPlugins();
    inputStoreOptions = CLIUtils.loadStore(configProperties, inputStoreName, configFile, console);
    final IndexStore indexStore = inputStoreOptions.createIndexStore();
    indices = DataStoreUtils.loadIndices(indexStore, indexList);
    // first collect the local file ingest plugins
    final Map<String, LocalFileIngestPlugin<?>> localFileIngestPlugins = new HashMap<>();
    final List<DataTypeAdapter<?>> adapters = new ArrayList<>();
    for (final Entry<String, LocalFileIngestPlugin<?>> pluginEntry : ingestPlugins.entrySet()) {
        if (!IngestUtils.checkIndexesAgainstProvider(pluginEntry.getKey(), pluginEntry.getValue(), indices)) {
            continue;
        }
        localFileIngestPlugins.put(pluginEntry.getKey(), pluginEntry.getValue());
        adapters.addAll(Arrays.asList(pluginEntry.getValue().getDataAdapters()));
    }
    final LocalFileIngestCLIDriver localIngestDriver = new LocalFileIngestCLIDriver(inputStoreOptions, indices, localFileIngestPlugins, visibilityOptions, localInput, 1);
    localIngestDriver.startExecutor();
    final DataStore dataStore = inputStoreOptions.createDataStore();
    try (LocalIngestRunData runData = new LocalIngestRunData(adapters, dataStore, visibilityOptions.getConfiguredVisibilityHandler())) {
        final List<PluginVisitor<LocalFileIngestPlugin<?>>> pluginVisitors = new ArrayList<>(localFileIngestPlugins.size());
        for (final Entry<String, LocalFileIngestPlugin<?>> localPlugin : localFileIngestPlugins.entrySet()) {
            pluginVisitors.add(new PluginVisitor<LocalFileIngestPlugin<?>>(localPlugin.getValue(), localPlugin.getKey(), localInput.getExtensions()));
        }
        while (inputFiles.hasNext()) {
            final URL file = inputFiles.next().toURL();
            for (final PluginVisitor<LocalFileIngestPlugin<?>> visitor : pluginVisitors) {
                if (visitor.supportsFile(file)) {
                    localIngestDriver.processFile(file, visitor.getTypeName(), visitor.getLocalPluginBase(), runData);
                }
            }
        }
    } catch (final MalformedURLException e) {
        LOGGER.error("Error in converting input path to URL for " + inputFiles, e);
        throw new MalformedURLException("Error in converting input path to URL for " + inputFiles);
    } catch (final Exception e) {
        LOGGER.error("Error processing in processing input", e);
        throw new RuntimeException("Error processing in processing input", e);
    } finally {
        localIngestDriver.shutdownExecutor();
    }
}