Examples with MapReduceDataStoreOperations org.locationtech.geowave.mapreduce.MapReduceDataStoreOperations used on opensource projects

Example 1 with MapReduceDataStoreOperations

use of org.locationtech.geowave.mapreduce.MapReduceDataStoreOperations in project geowave by locationtech.

the class SplitsProviderIT method getSplitsMSE.

private double getSplitsMSE(final QueryConstraints query, final int minSplits, final int maxSplits) {
    // get splits and create reader for each RangeLocationPair, then summing
    // up the rows for each split
    List<InputSplit> splits = null;
    final MapReduceDataStore dataStore = (MapReduceDataStore) dataStorePluginOptions.createDataStore();
    final PersistentAdapterStore as = dataStorePluginOptions.createAdapterStore();
    final InternalAdapterStore ias = dataStorePluginOptions.createInternalAdapterStore();
    final MapReduceDataStoreOperations ops = (MapReduceDataStoreOperations) dataStorePluginOptions.createDataStoreOperations();
    final IndexStore is = dataStorePluginOptions.createIndexStore();
    final AdapterIndexMappingStore aim = dataStorePluginOptions.createAdapterIndexMappingStore();
    final DataStatisticsStore stats = dataStorePluginOptions.createDataStatisticsStore();
    final MemoryAdapterStore mas = new MemoryAdapterStore();
    mas.addAdapter(fda);
    try {
        splits = dataStore.getSplits(new CommonQueryOptions(), new FilterByTypeQueryOptions<>(new String[] { fda.getTypeName() }), new QuerySingleIndex(idx.getName()), new EverythingQuery(), mas, aim, stats, ias, is, new JobContextImpl(new Configuration(), new JobID()), minSplits, maxSplits);
    } catch (final IOException e) {
        LOGGER.error("IOException thrown when calling getSplits", e);
    } catch (final InterruptedException e) {
        LOGGER.error("InterruptedException thrown when calling getSplits", e);
    }
    final double[] observed = new double[splits.size()];
    int totalCount = 0;
    int currentSplit = 0;
    for (final InputSplit split : splits) {
        int countPerSplit = 0;
        if (GeoWaveInputSplit.class.isAssignableFrom(split.getClass())) {
            final GeoWaveInputSplit gwSplit = (GeoWaveInputSplit) split;
            for (final String indexName : gwSplit.getIndexNames()) {
                final SplitInfo splitInfo = gwSplit.getInfo(indexName);
                for (final RangeLocationPair p : splitInfo.getRangeLocationPairs()) {
                    final RecordReaderParams readerParams = new RecordReaderParams(splitInfo.getIndex(), as, aim, ias, new short[] { ias.getAdapterId(fda.getTypeName()) }, null, null, null, splitInfo.isMixedVisibility(), splitInfo.isAuthorizationsLimiting(), splitInfo.isClientsideRowMerging(), p.getRange(), null, null);
                    try (RowReader<?> reader = ops.createReader(readerParams)) {
                        while (reader.hasNext()) {
                            reader.next();
                            countPerSplit++;
                        }
                    } catch (final Exception e) {
                        LOGGER.error("Exception thrown when calling createReader", e);
                    }
                }
            }
        }
        totalCount += countPerSplit;
        observed[currentSplit] = countPerSplit;
        currentSplit++;
    }
    final double expected = 1.0 / splits.size();
    double sum = 0;
    for (int i = 0; i < observed.length; i++) {
        sum += Math.pow((observed[i] / totalCount) - expected, 2);
    }
    return sum / splits.size();
}