Examples with PartitionsValue org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic.PartitionsValue used on opensource projects

Example 1 with PartitionsValue

use of org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic.PartitionsValue in project geowave by locationtech.

the class HBaseSplitsProvider method populateIntermediateSplits.

@Override
protected TreeSet<IntermediateSplitInfo> populateIntermediateSplits(final TreeSet<IntermediateSplitInfo> splits, final DataStoreOperations operations, final Index index, final List<Short> adapterIds, final Map<Pair<Index, ByteArray>, RowRangeHistogramValue> statsCache, final TransientAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final DataStatisticsStore statsStore, final Integer maxSplits, final QueryConstraints query, final double[] targetResolutionPerDimensionForHierarchicalIndex, final IndexMetaData[] indexMetadata, final String[] authorizations) throws IOException {
    HBaseOperations hbaseOperations = null;
    if (operations instanceof HBaseOperations) {
        hbaseOperations = (HBaseOperations) operations;
    } else {
        LOGGER.error("HBaseSplitsProvider requires BasicHBaseOperations object.");
        return splits;
    }
    final String tableName = hbaseOperations.getQualifiedTableName(index.getName());
    final Map<HRegionLocation, Map<HRegionInfo, List<ByteArrayRange>>> binnedRanges = new HashMap<>();
    final RegionLocator regionLocator = hbaseOperations.getRegionLocator(tableName);
    if (regionLocator == null) {
        LOGGER.error("Unable to retrieve RegionLocator for " + tableName);
        return splits;
    }
    // Build list of row ranges from query
    List<ByteArrayRange> ranges = null;
    if (query != null) {
        final List<MultiDimensionalNumericData> indexConstraints = query.getIndexConstraints(index);
        if ((maxSplits != null) && (maxSplits > 0)) {
            ranges = DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, maxSplits, indexMetadata).getCompositeQueryRanges();
        } else {
            ranges = DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, -1, indexMetadata).getCompositeQueryRanges();
        }
    }
    PersistentAdapterStore persistentAdapterStore = new AdapterStoreWrapper(adapterStore, internalAdapterStore);
    if (ranges == null) {
        // get partition ranges from stats
        final PartitionsValue statistics = InternalStatisticsHelper.getPartitions(index, adapterIds, persistentAdapterStore, statsStore, authorizations);
        if (statistics != null) {
            ranges = Lists.newArrayList();
            byte[] prevKey = HConstants.EMPTY_BYTE_ARRAY;
            final TreeSet<ByteArray> sortedPartitions = new TreeSet<>(statistics.getValue());
            for (final ByteArray partitionKey : sortedPartitions) {
                final ByteArrayRange range = new ByteArrayRange(prevKey, partitionKey.getBytes());
                ranges.add(range);
                prevKey = partitionKey.getBytes();
            }
            ranges.add(new ByteArrayRange(prevKey, HConstants.EMPTY_BYTE_ARRAY));
            binRanges(ranges, binnedRanges, regionLocator);
        } else {
            binFullRange(binnedRanges, regionLocator);
        }
    } else {
        while (!ranges.isEmpty()) {
            ranges = binRanges(ranges, binnedRanges, regionLocator);
        }
    }
    for (final Entry<HRegionLocation, Map<HRegionInfo, List<ByteArrayRange>>> locationEntry : binnedRanges.entrySet()) {
        final String hostname = locationEntry.getKey().getHostname();
        for (final Entry<HRegionInfo, List<ByteArrayRange>> regionEntry : locationEntry.getValue().entrySet()) {
            final Map<String, SplitInfo> splitInfo = new HashMap<>();
            final List<RangeLocationPair> rangeList = new ArrayList<>();
            for (final ByteArrayRange range : regionEntry.getValue()) {
                final GeoWaveRowRange gwRange = toRowRange(range, index.getIndexStrategy().getPartitionKeyLength());
                final double cardinality = getCardinality(getHistStats(index, adapterIds, persistentAdapterStore, statsStore, statsCache, new ByteArray(gwRange.getPartitionKey()), authorizations), gwRange);
                rangeList.add(new RangeLocationPair(gwRange, hostname, cardinality < 1 ? 1.0 : cardinality));
            }
            if (!rangeList.isEmpty()) {
                splitInfo.put(index.getName(), new SplitInfo(index, rangeList));
                splits.add(new IntermediateSplitInfo(splitInfo, this));
            }
        }
    }
    return splits;
}

Example 2 with PartitionsValue

use of org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic.PartitionsValue in project geowave by locationtech.

the class SplitsProvider method populateIntermediateSplits.

protected TreeSet<IntermediateSplitInfo> populateIntermediateSplits(final TreeSet<IntermediateSplitInfo> splits, final DataStoreOperations operations, final Index index, final List<Short> adapterIds, final Map<Pair<Index, ByteArray>, RowRangeHistogramValue> statsCache, final TransientAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final DataStatisticsStore statsStore, final Integer maxSplits, final QueryConstraints constraints, final double[] targetResolutionPerDimensionForHierarchicalIndex, final IndexMetaData[] indexMetadata, final String[] authorizations) throws IOException {
    // Build list of row ranges from query
    List<ByteArrayRange> ranges = null;
    if (constraints != null) {
        final List<MultiDimensionalNumericData> indexConstraints = constraints.getIndexConstraints(index);
        if ((maxSplits != null) && (maxSplits > 0)) {
            ranges = DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, maxSplits, indexMetadata).getCompositeQueryRanges();
        } else {
            ranges = DataStoreUtils.constraintsToQueryRanges(indexConstraints, index, targetResolutionPerDimensionForHierarchicalIndex, -1, indexMetadata).getCompositeQueryRanges();
        }
    }
    final List<RangeLocationPair> rangeList = new ArrayList<>();
    final PersistentAdapterStore persistentAdapterStore = new AdapterStoreWrapper(adapterStore, internalAdapterStore);
    if (ranges == null) {
        final PartitionsValue statistics = InternalStatisticsHelper.getPartitions(index, adapterIds, persistentAdapterStore, statsStore, authorizations);
        // Try to get ranges from histogram statistics
        if (statistics != null) {
            final Set<ByteArray> partitionKeys = statistics.getValue();
            for (final ByteArray partitionKey : partitionKeys) {
                final GeoWaveRowRange gwRange = new GeoWaveRowRange(partitionKey.getBytes(), null, null, true, true);
                final double cardinality = getCardinality(getHistStats(index, adapterIds, persistentAdapterStore, statsStore, statsCache, partitionKey, authorizations), gwRange);
                rangeList.add(new RangeLocationPair(gwRange, cardinality <= 0 ? 0 : cardinality < 1 ? 1.0 : cardinality));
            }
        } else {
            // add one all-inclusive range
            rangeList.add(new RangeLocationPair(new GeoWaveRowRange(null, null, null, true, false), 0.0));
        }
    } else {
        for (final ByteArrayRange range : ranges) {
            final GeoWaveRowRange gwRange = SplitsProvider.toRowRange(range, index.getIndexStrategy().getPartitionKeyLength());
            final double cardinality = getCardinality(getHistStats(index, adapterIds, persistentAdapterStore, statsStore, statsCache, new ByteArray(gwRange.getPartitionKey()), authorizations), gwRange);
            rangeList.add(new RangeLocationPair(gwRange, cardinality <= 0 ? 0 : cardinality < 1 ? 1.0 : cardinality));
        }
    }
    final Map<String, SplitInfo> splitInfo = new HashMap<>();
    if (!rangeList.isEmpty()) {
        splitInfo.put(index.getName(), new SplitInfo(index, rangeList));
        splits.add(new IntermediateSplitInfo(splitInfo, this));
    }
    return splits;
}

Example 3 with PartitionsValue

use of org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic.PartitionsValue in project geowave by locationtech.

the class PartitionsStatisticTest method testIngest.

@Test
public void testIngest() {
    final PartitionsStatistic statistic = new PartitionsStatistic();
    final PartitionsValue value = statistic.createEmpty();
    for (long i = 0; i < 10000; i++) {
        final GeoWaveRow row = new GeoWaveRowImpl(genKey(i), new GeoWaveValue[] {});
        value.entryIngested(null, 1, row);
    }
    assertEquals(32, value.getValue().size());
    for (byte i = 0; i < 32; i++) {
        Assert.assertTrue(value.getValue().contains(new ByteArray(new byte[] { i })));
    }
}

Example 4 with PartitionsValue

use of org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic.PartitionsValue in project geowave by locationtech.

the class DeletePyramidLevelCommand method run.

public void run(final OperationParams params) {
    // Ensure we have all the required arguments
    if (parameters.size() != 1) {
        throw new ParameterException("Requires argument: <store name>");
    }
    final String inputStoreName = parameters.get(0);
    // Attempt to load store.
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, getGeoWaveConfigFile(params), params.getConsole());
    final DataStore store = inputStoreOptions.createDataStore();
    RasterDataAdapter adapter = null;
    for (final DataTypeAdapter<?> type : store.getTypes()) {
        if (isRaster(type) && ((coverageName == null) || coverageName.equals(adapter.getTypeName()))) {
            if (adapter != null) {
                LOGGER.error("Store has multiple coverages.  Must explicitly choose one with --coverage option.");
                return;
            }
            adapter = (RasterDataAdapter) type;
        }
    }
    if (adapter == null) {
        LOGGER.error("Store has no coverages or coverage name not found.");
        return;
    }
    boolean found = false;
    Resolution res = null;
    Index i = null;
    for (final Index index : store.getIndices(adapter.getTypeName())) {
        final HierarchicalNumericIndexStrategy indexStrategy = CompoundHierarchicalIndexStrategyWrapper.findHierarchicalStrategy(index.getIndexStrategy());
        if (indexStrategy != null) {
            for (final SubStrategy s : indexStrategy.getSubStrategies()) {
                if ((s.getPrefix().length == 1) && (s.getPrefix()[0] == level)) {
                    LOGGER.info("Deleting from index " + index.getName());
                    final double[] tileRes = s.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                    final double[] pixelRes = new double[tileRes.length];
                    for (int d = 0; d < tileRes.length; d++) {
                        pixelRes[d] = tileRes[d] / adapter.getTileSize();
                    }
                    found = true;
                    i = index;
                    res = new Resolution(pixelRes);
                    break;
                }
            }
        }
        if (found) {
            break;
        }
    }
    if (!found) {
        LOGGER.error("Store has no indices supporting pyramids.");
        return;
    }
    final byte[][] predefinedSplits = i.getIndexStrategy().getPredefinedSplits();
    // this should account for hash partitioning if used
    final List<ByteArray> partitions = new ArrayList<>();
    if ((predefinedSplits != null) && (predefinedSplits.length > 0)) {
        for (final byte[] split : predefinedSplits) {
            partitions.add(new ByteArray(ArrayUtils.add(split, level.byteValue())));
        }
    } else {
        partitions.add(new ByteArray(new byte[] { level.byteValue() }));
    }
    // delete the resolution from the overview, delete the partitions, and delete the data
    if (inputStoreOptions.getFactoryOptions().getStoreOptions().isPersistDataStatistics()) {
        final DataStatisticsStore statsStore = inputStoreOptions.createDataStatisticsStore();
        boolean overviewStatsFound = false;
        boolean partitionStatsFound = false;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getDataTypeStatistics(adapter, RasterOverviewStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if ((next instanceof RasterOverviewStatistic) && (next.getBinningStrategy() == null)) {
                    final RasterOverviewStatistic statistic = (RasterOverviewStatistic) next;
                    final RasterOverviewValue value = statsStore.getStatisticValue(statistic);
                    if (!value.removeResolution(res)) {
                        LOGGER.error("Unable to remove resolution for pyramid level " + level);
                        return;
                    }
                    statsStore.setStatisticValue(statistic, value);
                    overviewStatsFound = true;
                }
            }
        }
        if (!overviewStatsFound) {
            LOGGER.error("Unable to find overview stats for coverage " + adapter.getTypeName());
            return;
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getIndexStatistics(i, PartitionsStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if (next instanceof PartitionsStatistic) {
                    if ((next.getBinningStrategy() != null) && (next.getBinningStrategy() instanceof DataTypeBinningStrategy)) {
                        final PartitionsStatistic statistic = (PartitionsStatistic) next;
                        final PartitionsValue value = statsStore.getStatisticValue((PartitionsStatistic) next, DataTypeBinningStrategy.getBin(adapter));
                        for (final ByteArray p : partitions) {
                            if (!value.getValue().remove(p)) {
                                LOGGER.error("Unable to remove partition " + p.getHexString() + " for pyramid level " + level);
                                return;
                            }
                        }
                        statsStore.setStatisticValue(statistic, value, DataTypeBinningStrategy.getBin(adapter));
                        partitionStatsFound = true;
                    }
                }
            }
        }
        if (!partitionStatsFound) {
            LOGGER.error("Unable to find partition stats for coverage " + adapter.getTypeName() + " and index " + i.getName());
            return;
        }
    }
    for (final ByteArray p : partitions) {
        store.delete(QueryBuilder.newBuilder().constraints(QueryBuilder.newBuilder().constraintsFactory().prefix(p.getBytes(), null)).addTypeName(adapter.getTypeName()).indexName(i.getName()).build());
    }
}