Examples with MapReduceDataStore org.locationtech.geowave.mapreduce.MapReduceDataStore used on opensource projects

Example 1 with MapReduceDataStore

use of org.locationtech.geowave.mapreduce.MapReduceDataStore in project geowave by locationtech.

the class GeoWaveInputFormat method getSplits.

@Override
public List<InputSplit> getSplits(final JobContext context) throws IOException, InterruptedException {
    final Map<String, String> configOptions = getStoreOptionsMap(context);
    final DataStore dataStore = GeoWaveStoreFinder.createDataStore(configOptions);
    if ((dataStore != null) && (dataStore instanceof MapReduceDataStore)) {
        return ((MapReduceDataStore) dataStore).getSplits(getCommonQueryOptions(context), getDataTypeQueryOptions(context), getIndexQueryOptions(context), getQueryConstraints(context), getJobContextAdapterStore(context), getJobContextAdapterIndexMappingStore(context), getJobContextDataStatisticsStore(context), getJobContextInternalAdapterStore(context), getJobContextIndexStore(context), context, getMinimumSplitCount(context), getMaximumSplitCount(context));
    }
    LOGGER.error("Data Store does not support map reduce");
    throw new IOException("Data Store does not support map reduce");
}

Example 2 with MapReduceDataStore

use of org.locationtech.geowave.mapreduce.MapReduceDataStore in project geowave by locationtech.

the class GeoWaveInputFormat method createRecordReader.

@Override
public RecordReader<GeoWaveInputKey, T> createRecordReader(final InputSplit split, final TaskAttemptContext context) throws IOException, InterruptedException {
    final Map<String, String> configOptions = getStoreOptionsMap(context);
    final DataStore dataStore = GeoWaveStoreFinder.createDataStore(configOptions);
    if ((dataStore != null) && (dataStore instanceof MapReduceDataStore)) {
        return (RecordReader<GeoWaveInputKey, T>) ((MapReduceDataStore) dataStore).createRecordReader(getCommonQueryOptions(context), getDataTypeQueryOptions(context), getIndexQueryOptions(context), getQueryConstraints(context), getJobContextAdapterStore(context), getJobContextInternalAdapterStore(context), getJobContextAdapterIndexMappingStore(context), getJobContextDataStatisticsStore(context), getJobContextIndexStore(context), isOutputWritable(context).booleanValue(), split);
    }
    LOGGER.error("Data Store does not support map reduce");
    throw new IOException("Data Store does not support map reduce");
}

Example 3 with MapReduceDataStore

use of org.locationtech.geowave.mapreduce.MapReduceDataStore in project geowave by locationtech.

the class GeoWaveOutputFormat method setStoreOptions.

public static void setStoreOptions(final Configuration config, final DataStorePluginOptions storeOptions) {
    if (storeOptions != null) {
        GeoWaveConfiguratorBase.setStoreOptionsMap(CLASS, config, storeOptions.getOptionsAsMap());
        final DataStore dataStore = storeOptions.createDataStore();
        if ((dataStore != null) && (dataStore instanceof MapReduceDataStore)) {
            ((MapReduceDataStore) dataStore).prepareRecordWriter(config);
        }
    } else {
        GeoWaveConfiguratorBase.setStoreOptionsMap(CLASS, config, null);
    }
}

Example 4 with MapReduceDataStore

use of org.locationtech.geowave.mapreduce.MapReduceDataStore in project geowave by locationtech.

the class SplitsProviderIT method getSplitsMSE.

private double getSplitsMSE(final QueryConstraints query, final int minSplits, final int maxSplits) {
    // get splits and create reader for each RangeLocationPair, then summing
    // up the rows for each split
    List<InputSplit> splits = null;
    final MapReduceDataStore dataStore = (MapReduceDataStore) dataStorePluginOptions.createDataStore();
    final PersistentAdapterStore as = dataStorePluginOptions.createAdapterStore();
    final InternalAdapterStore ias = dataStorePluginOptions.createInternalAdapterStore();
    final MapReduceDataStoreOperations ops = (MapReduceDataStoreOperations) dataStorePluginOptions.createDataStoreOperations();
    final IndexStore is = dataStorePluginOptions.createIndexStore();
    final AdapterIndexMappingStore aim = dataStorePluginOptions.createAdapterIndexMappingStore();
    final DataStatisticsStore stats = dataStorePluginOptions.createDataStatisticsStore();
    final MemoryAdapterStore mas = new MemoryAdapterStore();
    mas.addAdapter(fda);
    try {
        splits = dataStore.getSplits(new CommonQueryOptions(), new FilterByTypeQueryOptions<>(new String[] { fda.getTypeName() }), new QuerySingleIndex(idx.getName()), new EverythingQuery(), mas, aim, stats, ias, is, new JobContextImpl(new Configuration(), new JobID()), minSplits, maxSplits);
    } catch (final IOException e) {
        LOGGER.error("IOException thrown when calling getSplits", e);
    } catch (final InterruptedException e) {
        LOGGER.error("InterruptedException thrown when calling getSplits", e);
    }
    final double[] observed = new double[splits.size()];
    int totalCount = 0;
    int currentSplit = 0;
    for (final InputSplit split : splits) {
        int countPerSplit = 0;
        if (GeoWaveInputSplit.class.isAssignableFrom(split.getClass())) {
            final GeoWaveInputSplit gwSplit = (GeoWaveInputSplit) split;
            for (final String indexName : gwSplit.getIndexNames()) {
                final SplitInfo splitInfo = gwSplit.getInfo(indexName);
                for (final RangeLocationPair p : splitInfo.getRangeLocationPairs()) {
                    final RecordReaderParams readerParams = new RecordReaderParams(splitInfo.getIndex(), as, aim, ias, new short[] { ias.getAdapterId(fda.getTypeName()) }, null, null, null, splitInfo.isMixedVisibility(), splitInfo.isAuthorizationsLimiting(), splitInfo.isClientsideRowMerging(), p.getRange(), null, null);
                    try (RowReader<?> reader = ops.createReader(readerParams)) {
                        while (reader.hasNext()) {
                            reader.next();
                            countPerSplit++;
                        }
                    } catch (final Exception e) {
                        LOGGER.error("Exception thrown when calling createReader", e);
                    }
                }
            }
        }
        totalCount += countPerSplit;
        observed[currentSplit] = countPerSplit;
        currentSplit++;
    }
    final double expected = 1.0 / splits.size();
    double sum = 0;
    for (int i = 0; i < observed.length; i++) {
        sum += Math.pow((observed[i] / totalCount) - expected, 2);
    }
    return sum / splits.size();
}