Search in sources :

Example 1 with MapReduceDataStoreOperations

use of org.locationtech.geowave.mapreduce.MapReduceDataStoreOperations in project geowave by locationtech.

the class SplitsProviderIT method getSplitsMSE.

private double getSplitsMSE(final QueryConstraints query, final int minSplits, final int maxSplits) {
    // get splits and create reader for each RangeLocationPair, then summing
    // up the rows for each split
    List<InputSplit> splits = null;
    final MapReduceDataStore dataStore = (MapReduceDataStore) dataStorePluginOptions.createDataStore();
    final PersistentAdapterStore as = dataStorePluginOptions.createAdapterStore();
    final InternalAdapterStore ias = dataStorePluginOptions.createInternalAdapterStore();
    final MapReduceDataStoreOperations ops = (MapReduceDataStoreOperations) dataStorePluginOptions.createDataStoreOperations();
    final IndexStore is = dataStorePluginOptions.createIndexStore();
    final AdapterIndexMappingStore aim = dataStorePluginOptions.createAdapterIndexMappingStore();
    final DataStatisticsStore stats = dataStorePluginOptions.createDataStatisticsStore();
    final MemoryAdapterStore mas = new MemoryAdapterStore();
    mas.addAdapter(fda);
    try {
        splits = dataStore.getSplits(new CommonQueryOptions(), new FilterByTypeQueryOptions<>(new String[] { fda.getTypeName() }), new QuerySingleIndex(idx.getName()), new EverythingQuery(), mas, aim, stats, ias, is, new JobContextImpl(new Configuration(), new JobID()), minSplits, maxSplits);
    } catch (final IOException e) {
        LOGGER.error("IOException thrown when calling getSplits", e);
    } catch (final InterruptedException e) {
        LOGGER.error("InterruptedException thrown when calling getSplits", e);
    }
    final double[] observed = new double[splits.size()];
    int totalCount = 0;
    int currentSplit = 0;
    for (final InputSplit split : splits) {
        int countPerSplit = 0;
        if (GeoWaveInputSplit.class.isAssignableFrom(split.getClass())) {
            final GeoWaveInputSplit gwSplit = (GeoWaveInputSplit) split;
            for (final String indexName : gwSplit.getIndexNames()) {
                final SplitInfo splitInfo = gwSplit.getInfo(indexName);
                for (final RangeLocationPair p : splitInfo.getRangeLocationPairs()) {
                    final RecordReaderParams readerParams = new RecordReaderParams(splitInfo.getIndex(), as, aim, ias, new short[] { ias.getAdapterId(fda.getTypeName()) }, null, null, null, splitInfo.isMixedVisibility(), splitInfo.isAuthorizationsLimiting(), splitInfo.isClientsideRowMerging(), p.getRange(), null, null);
                    try (RowReader<?> reader = ops.createReader(readerParams)) {
                        while (reader.hasNext()) {
                            reader.next();
                            countPerSplit++;
                        }
                    } catch (final Exception e) {
                        LOGGER.error("Exception thrown when calling createReader", e);
                    }
                }
            }
        }
        totalCount += countPerSplit;
        observed[currentSplit] = countPerSplit;
        currentSplit++;
    }
    final double expected = 1.0 / splits.size();
    double sum = 0;
    for (int i = 0; i < observed.length; i++) {
        sum += Math.pow((observed[i] / totalCount) - expected, 2);
    }
    return sum / splits.size();
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) CommonQueryOptions(org.locationtech.geowave.core.store.query.options.CommonQueryOptions) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) QuerySingleIndex(org.locationtech.geowave.core.store.query.options.QuerySingleIndex) InputSplit(org.apache.hadoop.mapreduce.InputSplit) MapReduceDataStoreOperations(org.locationtech.geowave.mapreduce.MapReduceDataStoreOperations) JobContextImpl(org.apache.hadoop.mapreduce.task.JobContextImpl) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) EverythingQuery(org.locationtech.geowave.core.store.query.constraints.EverythingQuery) MemoryAdapterStore(org.locationtech.geowave.core.store.memory.MemoryAdapterStore) IOException(java.io.IOException) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) IOException(java.io.IOException) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) MapReduceDataStore(org.locationtech.geowave.mapreduce.MapReduceDataStore) FilterByTypeQueryOptions(org.locationtech.geowave.core.store.query.options.FilterByTypeQueryOptions) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) JobID(org.apache.hadoop.mapreduce.JobID)

Aggregations

IOException (java.io.IOException)1 Configuration (org.apache.hadoop.conf.Configuration)1 InputSplit (org.apache.hadoop.mapreduce.InputSplit)1 JobID (org.apache.hadoop.mapreduce.JobID)1 JobContextImpl (org.apache.hadoop.mapreduce.task.JobContextImpl)1 AdapterIndexMappingStore (org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore)1 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)1 PersistentAdapterStore (org.locationtech.geowave.core.store.adapter.PersistentAdapterStore)1 IndexStore (org.locationtech.geowave.core.store.index.IndexStore)1 MemoryAdapterStore (org.locationtech.geowave.core.store.memory.MemoryAdapterStore)1 EverythingQuery (org.locationtech.geowave.core.store.query.constraints.EverythingQuery)1 CommonQueryOptions (org.locationtech.geowave.core.store.query.options.CommonQueryOptions)1 FilterByTypeQueryOptions (org.locationtech.geowave.core.store.query.options.FilterByTypeQueryOptions)1 QuerySingleIndex (org.locationtech.geowave.core.store.query.options.QuerySingleIndex)1 DataStatisticsStore (org.locationtech.geowave.core.store.statistics.DataStatisticsStore)1 MapReduceDataStore (org.locationtech.geowave.mapreduce.MapReduceDataStore)1 MapReduceDataStoreOperations (org.locationtech.geowave.mapreduce.MapReduceDataStoreOperations)1