Examples with LocalFileIngestPlugin org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin used on opensource projects

Example 1 with LocalFileIngestPlugin

use of org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}

Example 2 with LocalFileIngestPlugin

use of org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin in project geowave by locationtech.

the class IngestFormatPluginOptions method createLocalIngestPlugins.

public Map<String, LocalFileIngestPlugin<?>> createLocalIngestPlugins() {
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = new HashMap<>();
    for (final Entry<String, IngestFormatPluginProviderSpi<?, ?>> entry : plugins.entrySet()) {
        final IngestFormatPluginProviderSpi<?, ?> formatPlugin = entry.getValue();
        final IngestFormatOptions formatOptions = options.get(entry.getKey());
        LocalFileIngestPlugin<?> plugin = null;
        try {
            plugin = formatPlugin.createLocalFileIngestPlugin(formatOptions);
            if (plugin == null) {
                throw new UnsupportedOperationException();
            }
        } catch (final UnsupportedOperationException e) {
            LOGGER.warn("Plugin provider for ingest type '" + formatPlugin.getIngestFormatName() + "' does not support local file ingest", e);
            continue;
        }
        ingestPlugins.put(formatPlugin.getIngestFormatName(), plugin);
    }
    return ingestPlugins;
}

Example 3 with LocalFileIngestPlugin

use of org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin in project geowave by locationtech.

the class SparkIngestDriver method processInput.

public void processInput(final File configFile, final LocalInputCommandLineOptions localInput, final String inputStoreName, final String indexList, final VisibilityOptions visibilityOptions, final Properties configProperties, final Iterator<URI> inputFiles, final Console console) throws IOException {
    // Based on the selected formats, select the format plugins
    final IngestFormatPluginOptions pluginFormats = new IngestFormatPluginOptions();
    // Based on the selected formats, select the format plugins
    pluginFormats.selectPlugin(localInput.getFormats());
    DataStorePluginOptions inputStoreOptions = null;
    List<Index> indices = null;
    // Ingest Plugins
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = pluginFormats.createLocalIngestPlugins();
    inputStoreOptions = CLIUtils.loadStore(configProperties, inputStoreName, configFile, console);
    final IndexStore indexStore = inputStoreOptions.createIndexStore();
    indices = DataStoreUtils.loadIndices(indexStore, indexList);
    // first collect the local file ingest plugins
    final Map<String, LocalFileIngestPlugin<?>> localFileIngestPlugins = new HashMap<>();
    final List<DataTypeAdapter<?>> adapters = new ArrayList<>();
    for (final Entry<String, LocalFileIngestPlugin<?>> pluginEntry : ingestPlugins.entrySet()) {
        if (!IngestUtils.checkIndexesAgainstProvider(pluginEntry.getKey(), pluginEntry.getValue(), indices)) {
            continue;
        }
        localFileIngestPlugins.put(pluginEntry.getKey(), pluginEntry.getValue());
        adapters.addAll(Arrays.asList(pluginEntry.getValue().getDataAdapters()));
    }
    final LocalFileIngestCLIDriver localIngestDriver = new LocalFileIngestCLIDriver(inputStoreOptions, indices, localFileIngestPlugins, visibilityOptions, localInput, 1);
    localIngestDriver.startExecutor();
    final DataStore dataStore = inputStoreOptions.createDataStore();
    try (LocalIngestRunData runData = new LocalIngestRunData(adapters, dataStore, visibilityOptions.getConfiguredVisibilityHandler())) {
        final List<PluginVisitor<LocalFileIngestPlugin<?>>> pluginVisitors = new ArrayList<>(localFileIngestPlugins.size());
        for (final Entry<String, LocalFileIngestPlugin<?>> localPlugin : localFileIngestPlugins.entrySet()) {
            pluginVisitors.add(new PluginVisitor<LocalFileIngestPlugin<?>>(localPlugin.getValue(), localPlugin.getKey(), localInput.getExtensions()));
        }
        while (inputFiles.hasNext()) {
            final URL file = inputFiles.next().toURL();
            for (final PluginVisitor<LocalFileIngestPlugin<?>> visitor : pluginVisitors) {
                if (visitor.supportsFile(file)) {
                    localIngestDriver.processFile(file, visitor.getTypeName(), visitor.getLocalPluginBase(), runData);
                }
            }
        }
    } catch (final MalformedURLException e) {
        LOGGER.error("Error in converting input path to URL for " + inputFiles, e);
        throw new MalformedURLException("Error in converting input path to URL for " + inputFiles);
    } catch (final Exception e) {
        LOGGER.error("Error processing in processing input", e);
        throw new RuntimeException("Error processing in processing input", e);
    } finally {
        localIngestDriver.shutdownExecutor();
    }
}

Example 4 with LocalFileIngestPlugin

use of org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin in project geowave by locationtech.

the class LocalToKafkaCommand method computeResults.

@Override
public Void computeResults(final OperationParams params) throws Exception {
    // Ensure we have all the required arguments
    if (parameters.size() != 1) {
        throw new ParameterException("Requires arguments: <file or directory>");
    }
    final String inputPath = parameters.get(0);
    // Ingest Plugins
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = pluginFormats.createLocalIngestPlugins();
    // Driver
    final StageToKafkaDriver driver = new StageToKafkaDriver(kafkaOptions, ingestPlugins, localInputOptions);
    // Config file
    final File configFile = getGeoWaveConfigFile(params);
    // Execute
    if (!driver.runOperation(inputPath, configFile)) {
        throw new RuntimeException("Ingest failed to execute");
    }
    return null;
}

Example 5 with LocalFileIngestPlugin

use of org.locationtech.geowave.core.store.ingest.LocalFileIngestPlugin in project geowave by locationtech.

the class AddTypeCommand method getAllDataAdapters.

public List<DataTypeAdapter<?>> getAllDataAdapters(final String inputPath, final File configFile) throws IOException {
    final Map<String, LocalFileIngestPlugin<?>> ingestPlugins = pluginFormats.createLocalIngestPlugins();
    final Map<String, LocalFileIngestPlugin<?>> localFileIngestPlugins = new HashMap<>();
    final Map<String, DataTypeAdapter<?>> adapters = Maps.newHashMap();
    for (final Entry<String, LocalFileIngestPlugin<?>> pluginEntry : ingestPlugins.entrySet()) {
        if (!isSupported(pluginEntry.getKey(), pluginEntry.getValue())) {
            continue;
        }
        localFileIngestPlugins.put(pluginEntry.getKey(), pluginEntry.getValue());
        Arrays.stream(pluginEntry.getValue().getDataAdapters()).forEach(adapter -> {
            adapters.put(adapter.getTypeName(), adapter);
        });
    }
    Properties configProperties = null;
    if ((configFile != null) && configFile.exists()) {
        configProperties = ConfigOptions.loadProperties(configFile);
    }
    Path path = IngestUtils.handleIngestUrl(inputPath, configProperties);
    if (path == null) {
        final File f = new File(inputPath);
        if (!f.exists()) {
            LOGGER.error("Input file '" + f.getAbsolutePath() + "' does not exist");
            throw new IllegalArgumentException(inputPath + " does not exist");
        }
        path = Paths.get(inputPath);
    }
    for (final LocalPluginBase localPlugin : localFileIngestPlugins.values()) {
        localPlugin.init(path.toUri().toURL());
    }
    final DataAdapterFileVisitor fileURLs = new DataAdapterFileVisitor(localFileIngestPlugins, localInputOptions.getExtensions(), adapters);
    Files.walkFileTree(path, fileURLs);
    return Lists.newArrayList(adapters.values());
}