Examples with PartitionsStatistic org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic used on opensource projects

Example 1 with PartitionsStatistic

use of org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic in project geowave by locationtech.

the class IndexImpl method getDefaultStatistics.

@Override
public List<Statistic<? extends StatisticValue<?>>> getDefaultStatistics() {
    List<Statistic<? extends StatisticValue<?>>> statistics = Lists.newArrayListWithCapacity(6);
    IndexMetaDataSetStatistic metadata = new IndexMetaDataSetStatistic(getName(), indexStrategy.createMetaData());
    metadata.setBinningStrategy(new DataTypeBinningStrategy());
    metadata.setInternal();
    statistics.add(metadata);
    DuplicateEntryCountStatistic duplicateCounts = new DuplicateEntryCountStatistic(getName());
    duplicateCounts.setBinningStrategy(new DataTypeBinningStrategy());
    duplicateCounts.setInternal();
    statistics.add(duplicateCounts);
    PartitionsStatistic partitions = new PartitionsStatistic(getName());
    partitions.setBinningStrategy(new DataTypeBinningStrategy());
    partitions.setInternal();
    statistics.add(partitions);
    DifferingVisibilityCountStatistic differingFieldVisibility = new DifferingVisibilityCountStatistic(getName());
    differingFieldVisibility.setBinningStrategy(new DataTypeBinningStrategy());
    differingFieldVisibility.setInternal();
    statistics.add(differingFieldVisibility);
    FieldVisibilityCountStatistic fieldVisibilityCount = new FieldVisibilityCountStatistic(getName());
    fieldVisibilityCount.setBinningStrategy(new DataTypeBinningStrategy());
    fieldVisibilityCount.setInternal();
    statistics.add(fieldVisibilityCount);
    RowRangeHistogramStatistic rowRangeHistogram = new RowRangeHistogramStatistic(getName());
    rowRangeHistogram.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
    rowRangeHistogram.setInternal();
    statistics.add(rowRangeHistogram);
    return statistics;
}

Example 2 with PartitionsStatistic

use of org.locationtech.geowave.core.store.statistics.index.PartitionsStatistic in project geowave by locationtech.

the class DeletePyramidLevelCommand method run.

public void run(final OperationParams params) {
    // Ensure we have all the required arguments
    if (parameters.size() != 1) {
        throw new ParameterException("Requires argument: <store name>");
    }
    final String inputStoreName = parameters.get(0);
    // Attempt to load store.
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, getGeoWaveConfigFile(params), params.getConsole());
    final DataStore store = inputStoreOptions.createDataStore();
    RasterDataAdapter adapter = null;
    for (final DataTypeAdapter<?> type : store.getTypes()) {
        if (isRaster(type) && ((coverageName == null) || coverageName.equals(adapter.getTypeName()))) {
            if (adapter != null) {
                LOGGER.error("Store has multiple coverages.  Must explicitly choose one with --coverage option.");
                return;
            }
            adapter = (RasterDataAdapter) type;
        }
    }
    if (adapter == null) {
        LOGGER.error("Store has no coverages or coverage name not found.");
        return;
    }
    boolean found = false;
    Resolution res = null;
    Index i = null;
    for (final Index index : store.getIndices(adapter.getTypeName())) {
        final HierarchicalNumericIndexStrategy indexStrategy = CompoundHierarchicalIndexStrategyWrapper.findHierarchicalStrategy(index.getIndexStrategy());
        if (indexStrategy != null) {
            for (final SubStrategy s : indexStrategy.getSubStrategies()) {
                if ((s.getPrefix().length == 1) && (s.getPrefix()[0] == level)) {
                    LOGGER.info("Deleting from index " + index.getName());
                    final double[] tileRes = s.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                    final double[] pixelRes = new double[tileRes.length];
                    for (int d = 0; d < tileRes.length; d++) {
                        pixelRes[d] = tileRes[d] / adapter.getTileSize();
                    }
                    found = true;
                    i = index;
                    res = new Resolution(pixelRes);
                    break;
                }
            }
        }
        if (found) {
            break;
        }
    }
    if (!found) {
        LOGGER.error("Store has no indices supporting pyramids.");
        return;
    }
    final byte[][] predefinedSplits = i.getIndexStrategy().getPredefinedSplits();
    // this should account for hash partitioning if used
    final List<ByteArray> partitions = new ArrayList<>();
    if ((predefinedSplits != null) && (predefinedSplits.length > 0)) {
        for (final byte[] split : predefinedSplits) {
            partitions.add(new ByteArray(ArrayUtils.add(split, level.byteValue())));
        }
    } else {
        partitions.add(new ByteArray(new byte[] { level.byteValue() }));
    }
    // delete the resolution from the overview, delete the partitions, and delete the data
    if (inputStoreOptions.getFactoryOptions().getStoreOptions().isPersistDataStatistics()) {
        final DataStatisticsStore statsStore = inputStoreOptions.createDataStatisticsStore();
        boolean overviewStatsFound = false;
        boolean partitionStatsFound = false;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getDataTypeStatistics(adapter, RasterOverviewStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if ((next instanceof RasterOverviewStatistic) && (next.getBinningStrategy() == null)) {
                    final RasterOverviewStatistic statistic = (RasterOverviewStatistic) next;
                    final RasterOverviewValue value = statsStore.getStatisticValue(statistic);
                    if (!value.removeResolution(res)) {
                        LOGGER.error("Unable to remove resolution for pyramid level " + level);
                        return;
                    }
                    statsStore.setStatisticValue(statistic, value);
                    overviewStatsFound = true;
                }
            }
        }
        if (!overviewStatsFound) {
            LOGGER.error("Unable to find overview stats for coverage " + adapter.getTypeName());
            return;
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getIndexStatistics(i, PartitionsStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if (next instanceof PartitionsStatistic) {
                    if ((next.getBinningStrategy() != null) && (next.getBinningStrategy() instanceof DataTypeBinningStrategy)) {
                        final PartitionsStatistic statistic = (PartitionsStatistic) next;
                        final PartitionsValue value = statsStore.getStatisticValue((PartitionsStatistic) next, DataTypeBinningStrategy.getBin(adapter));
                        for (final ByteArray p : partitions) {
                            if (!value.getValue().remove(p)) {
                                LOGGER.error("Unable to remove partition " + p.getHexString() + " for pyramid level " + level);
                                return;
                            }
                        }
                        statsStore.setStatisticValue(statistic, value, DataTypeBinningStrategy.getBin(adapter));
                        partitionStatsFound = true;
                    }
                }
            }
        }
        if (!partitionStatsFound) {
            LOGGER.error("Unable to find partition stats for coverage " + adapter.getTypeName() + " and index " + i.getName());
            return;
        }
    }
    for (final ByteArray p : partitions) {
        store.delete(QueryBuilder.newBuilder().constraints(QueryBuilder.newBuilder().constraintsFactory().prefix(p.getBytes(), null)).addTypeName(adapter.getTypeName()).indexName(i.getName()).build());
    }
}