Examples with IntermediateSplitInfo org.locationtech.geowave.mapreduce.splits.IntermediateSplitInfo used on opensource projects

Example 1 with IntermediateSplitInfo

use of org.locationtech.geowave.mapreduce.splits.IntermediateSplitInfo in project geowave by locationtech.

the class HBaseSplitsProvider method populateIntermediateSplits.

@Override
protected TreeSet<IntermediateSplitInfo> populateIntermediateSplits(final TreeSet<IntermediateSplitInfo> splits, final DataStoreOperations operations, final Index index, final List<Short> adapterIds, final Map<Pair<Index, ByteArray>, RowRangeHistogramValue> statsCache, final TransientAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final DataStatisticsStore statsStore, final Integer maxSplits, final QueryConstraints query, final double[] targetResolutionPerDimensionForHierarchicalIndex, final IndexMetaData[] indexMetadata, final String[] authorizations) throws IOException {
    HBaseOperations hbaseOperations = null;
    if (operations instanceof HBaseOperations) {
        hbaseOperations = (HBaseOperations) operations;
    } else {
        LOGGER.error("HBaseSplitsProvider requires BasicHBaseOperations object.");
        return splits;
    }
    final String tableName = hbaseOperations.getQualifiedTableName(index.getName());
    final Map<HRegionLocation, Map<HRegionInfo, List<ByteArrayRange>>> binnedRanges = new HashMap<>();
    final RegionLocator regionLocator = hbaseOperations.getRegionLocator(tableName);
    if (regionLocator == null) {
        LOGGER.error("Unable to retrieve RegionLocator for " + tableName);
        return splits;
    }
    // Build list of row ranges from query
    List<ByteArrayRange> ranges = null;
    if (query != null) {
        final List<MultiDimensionalNumericData> indexConstraints = query.getIndexConstraints(index);
        if ((maxSplits != null) && (maxSplits > 0)) {
            ranges = DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, maxSplits, indexMetadata).getCompositeQueryRanges();
        } else {
            ranges = DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, -1, indexMetadata).getCompositeQueryRanges();
        }
    }
    PersistentAdapterStore persistentAdapterStore = new AdapterStoreWrapper(adapterStore, internalAdapterStore);
    if (ranges == null) {
        // get partition ranges from stats
        final PartitionsValue statistics = InternalStatisticsHelper.getPartitions(index, adapterIds, persistentAdapterStore, statsStore, authorizations);
        if (statistics != null) {
            ranges = Lists.newArrayList();
            byte[] prevKey = HConstants.EMPTY_BYTE_ARRAY;
            final TreeSet<ByteArray> sortedPartitions = new TreeSet<>(statistics.getValue());
            for (final ByteArray partitionKey : sortedPartitions) {
                final ByteArrayRange range = new ByteArrayRange(prevKey, partitionKey.getBytes());
                ranges.add(range);
                prevKey = partitionKey.getBytes();
            }
            ranges.add(new ByteArrayRange(prevKey, HConstants.EMPTY_BYTE_ARRAY));
            binRanges(ranges, binnedRanges, regionLocator);
        } else {
            binFullRange(binnedRanges, regionLocator);
        }
    } else {
        while (!ranges.isEmpty()) {
            ranges = binRanges(ranges, binnedRanges, regionLocator);
        }
    }
    for (final Entry<HRegionLocation, Map<HRegionInfo, List<ByteArrayRange>>> locationEntry : binnedRanges.entrySet()) {
        final String hostname = locationEntry.getKey().getHostname();
        for (final Entry<HRegionInfo, List<ByteArrayRange>> regionEntry : locationEntry.getValue().entrySet()) {
            final Map<String, SplitInfo> splitInfo = new HashMap<>();
            final List<RangeLocationPair> rangeList = new ArrayList<>();
            for (final ByteArrayRange range : regionEntry.getValue()) {
                final GeoWaveRowRange gwRange = toRowRange(range, index.getIndexStrategy().getPartitionKeyLength());
                final double cardinality = getCardinality(getHistStats(index, adapterIds, persistentAdapterStore, statsStore, statsCache, new ByteArray(gwRange.getPartitionKey()), authorizations), gwRange);
                rangeList.add(new RangeLocationPair(gwRange, hostname, cardinality < 1 ? 1.0 : cardinality));
            }
            if (!rangeList.isEmpty()) {
                splitInfo.put(index.getName(), new SplitInfo(index, rangeList));
                splits.add(new IntermediateSplitInfo(splitInfo, this));
            }
        }
    }
    return splits;
}

Example 2 with IntermediateSplitInfo

use of org.locationtech.geowave.mapreduce.splits.IntermediateSplitInfo in project geowave by locationtech.

the class AccumuloSplitsProvider method populateIntermediateSplits.

@Override
protected TreeSet<IntermediateSplitInfo> populateIntermediateSplits(final TreeSet<IntermediateSplitInfo> splits, final DataStoreOperations operations, final Index index, final List<Short> adapterIds, final Map<Pair<Index, ByteArray>, RowRangeHistogramValue> statsCache, final TransientAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final DataStatisticsStore statsStore, final Integer maxSplits, final QueryConstraints constraints, final double[] targetResolutionPerDimensionForHierarchicalIndex, final IndexMetaData[] indexMetadata, final String[] authorizations) throws IOException {
    AccumuloOperations accumuloOperations = null;
    if (operations instanceof AccumuloOperations) {
        accumuloOperations = (AccumuloOperations) operations;
    } else {
        LOGGER.error("AccumuloSplitsProvider requires AccumuloOperations object.");
        return splits;
    }
    final int partitionKeyLength = index.getIndexStrategy().getPartitionKeyLength();
    Range fullrange;
    try {
        fullrange = toAccumuloRange(new GeoWaveRowRange(null, null, null, true, true), partitionKeyLength);
    } catch (final Exception e) {
        fullrange = new Range();
        LOGGER.warn("Cannot ascertain the full range of the data", e);
    }
    final String tableName = AccumuloUtils.getQualifiedTableName(accumuloOperations.getTableNameSpace(), index.getName());
    final TreeSet<Range> ranges;
    if (constraints != null) {
        final List<MultiDimensionalNumericData> indexConstraints = constraints.getIndexConstraints(index);
        if ((maxSplits != null) && (maxSplits > 0)) {
            ranges = AccumuloUtils.byteArrayRangesToAccumuloRanges(DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, maxSplits, indexMetadata).getCompositeQueryRanges());
        } else {
            ranges = AccumuloUtils.byteArrayRangesToAccumuloRanges(DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, -1, indexMetadata).getCompositeQueryRanges());
        }
        if (ranges.size() == 1) {
            final Range range = ranges.first();
            if (range.isInfiniteStartKey() || range.isInfiniteStopKey()) {
                ranges.remove(range);
                ranges.add(fullrange.clip(range));
            }
        }
    } else {
        ranges = new TreeSet<>();
        ranges.add(fullrange);
        if (LOGGER.isTraceEnabled()) {
            LOGGER.trace("Protected range: " + fullrange);
        }
    }
    // get the metadata information for these ranges
    final HashMap<String, String> hostNameCache = getHostNameCache();
    final Connector conn = accumuloOperations.getConnector();
    Locations locations;
    try {
        locations = conn.tableOperations().locate(tableName, ranges);
    } catch (AccumuloException | AccumuloSecurityException | TableNotFoundException e) {
        throw new IOException("Unable to get Tablet Locations", e);
    }
    for (final Entry<TabletId, List<Range>> tabletIdRanges : locations.groupByTablet().entrySet()) {
        final TabletId tabletId = tabletIdRanges.getKey();
        final String tabletServer = locations.getTabletLocation(tabletId);
        final String ipAddress = tabletServer.split(":", 2)[0];
        String location = hostNameCache.get(ipAddress);
        // authentication
        if (location == null) {
            final InetAddress inetAddress = InetAddress.getByName(ipAddress);
            location = inetAddress.getHostName();
            hostNameCache.put(ipAddress, location);
        }
        final Range tabletRange = tabletId.toRange();
        final Map<String, SplitInfo> splitInfo = new HashMap<>();
        final List<RangeLocationPair> rangeList = new ArrayList<>();
        for (final Range range : tabletIdRanges.getValue()) {
            final Range clippedRange = tabletRange.clip(range);
            if (!(fullrange.beforeStartKey(clippedRange.getEndKey()) || fullrange.afterEndKey(clippedRange.getStartKey()))) {
                final GeoWaveRowRange rowRange = fromAccumuloRange(clippedRange, partitionKeyLength);
                final double cardinality = getCardinality(getHistStats(index, adapterIds, new AdapterStoreWrapper(adapterStore, internalAdapterStore), statsStore, statsCache, new ByteArray(rowRange.getPartitionKey()), authorizations), rowRange);
                rangeList.add(new RangeLocationPair(rowRange, location, cardinality < 1 ? 1.0 : cardinality));
            } else {
                LOGGER.info("Query split outside of range");
            }
            if (LOGGER.isTraceEnabled()) {
                LOGGER.warn("Clipped range: " + rangeList.get(rangeList.size() - 1).getRange());
            }
        }
        if (!rangeList.isEmpty()) {
            splitInfo.put(index.getName(), new SplitInfo(index, rangeList));
            splits.add(new IntermediateSplitInfo(splitInfo, this));
        }
    }
    return splits;
}