Examples with HierarchicalNumericIndexStrategy org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy used on opensource projects

Example 1 with HierarchicalNumericIndexStrategy

use of org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy in project geowave by locationtech.

the class CompoundHierarchicalIndexStrategyWrapper method findHierarchicalStrategy.

public static HierarchicalNumericIndexStrategy findHierarchicalStrategy(final NumericIndexStrategy indexStrategy) {
    final List<CompoundIndexStrategy> parentStrategies = new ArrayList<>();
    final HierarchicalNumericIndexStrategy firstHierarchicalStrategy = findHierarchicalStrategy(indexStrategy, parentStrategies);
    if (firstHierarchicalStrategy == null) {
        return null;
    } else if (parentStrategies.isEmpty()) {
        return firstHierarchicalStrategy;
    } else {
        return new CompoundHierarchicalIndexStrategyWrapper(parentStrategies, firstHierarchicalStrategy);
    }
}

Example 2 with HierarchicalNumericIndexStrategy

use of org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy in project geowave by locationtech.

the class CompoundHierarchicalIndexStrategyWrapper method findHierarchicalStrategy.

public static HierarchicalNumericIndexStrategy findHierarchicalStrategy(final NumericIndexStrategy indexStrategy, final List<CompoundIndexStrategy> parentStrategies) {
    if (indexStrategy instanceof HierarchicalNumericIndexStrategy) {
        return (HierarchicalNumericIndexStrategy) indexStrategy;
    }
    if (indexStrategy instanceof CompoundIndexStrategy) {
        final PartitionIndexStrategy<MultiDimensionalNumericData, MultiDimensionalNumericData> primaryIndex = ((CompoundIndexStrategy) indexStrategy).getPrimarySubStrategy();
        final NumericIndexStrategy secondaryIndex = ((CompoundIndexStrategy) indexStrategy).getSecondarySubStrategy();
        // warn if round robin is used
        if (primaryIndex instanceof RoundRobinKeyIndexStrategy) {
            LOGGER.warn("Round Robin partitioning won't work correctly with raster merge strategies");
        } else if (secondaryIndex instanceof RoundRobinKeyIndexStrategy) {
            LOGGER.warn("Round Robin partitioning won't work correctly with raster merge strategies");
        }
        final HierarchicalNumericIndexStrategy secondary = findHierarchicalStrategy(secondaryIndex);
        if (secondary != null) {
            // add it to beginning because we are recursing back from the
            // leaf strategy up to the parent
            parentStrategies.add(0, (CompoundIndexStrategy) indexStrategy);
            return secondary;
        }
    }
    return null;
}

Example 3 with HierarchicalNumericIndexStrategy

use of org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy in project geowave by locationtech.

the class DataStoreUtils method constraintsToQueryRanges.

public static QueryRanges constraintsToQueryRanges(final List<MultiDimensionalNumericData> constraints, final Index index, final double[] targetResolutionPerDimensionForHierarchicalIndex, final int maxRanges, final IndexMetaData... hints) {
    if ((index instanceof CustomIndex) && (constraints != null) && (constraints.size() == 1) && (constraints.get(0) instanceof InternalCustomConstraints)) {
        return ((CustomIndex) index).getQueryRanges(((InternalCustomConstraints) constraints.get(0)).getCustomConstraints());
    }
    NumericIndexStrategy indexStrategy = index.getIndexStrategy();
    SubStrategy targetIndexStrategy = null;
    if ((targetResolutionPerDimensionForHierarchicalIndex != null) && (targetResolutionPerDimensionForHierarchicalIndex.length == indexStrategy.getOrderedDimensionDefinitions().length)) {
        // determine the correct tier to query for the given resolution
        final HierarchicalNumericIndexStrategy strategy = CompoundHierarchicalIndexStrategyWrapper.findHierarchicalStrategy(indexStrategy);
        if (strategy != null) {
            final TreeMap<Double, SubStrategy> sortedStrategies = new TreeMap<>();
            for (final SubStrategy subStrategy : strategy.getSubStrategies()) {
                final double[] idRangePerDimension = subStrategy.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                double rangeSum = 0;
                for (final double range : idRangePerDimension) {
                    rangeSum += range;
                }
                // sort by the sum of the range in each dimension
                sortedStrategies.put(rangeSum, subStrategy);
            }
            for (final SubStrategy subStrategy : sortedStrategies.descendingMap().values()) {
                final double[] highestPrecisionIdRangePerDimension = subStrategy.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                // if the id range is less than or equal to the target
                // resolution in each dimension, use this substrategy
                boolean withinTargetResolution = true;
                for (int d = 0; d < highestPrecisionIdRangePerDimension.length; d++) {
                    if (highestPrecisionIdRangePerDimension[d] > targetResolutionPerDimensionForHierarchicalIndex[d]) {
                        withinTargetResolution = false;
                        break;
                    }
                }
                if (withinTargetResolution) {
                    targetIndexStrategy = subStrategy;
                    break;
                }
            }
            if (targetIndexStrategy == null) {
                // if there is not a substrategy that is within the target
                // resolution, use the first substrategy (the lowest range
                // per dimension, which is the highest precision)
                targetIndexStrategy = sortedStrategies.firstEntry().getValue();
            }
            indexStrategy = targetIndexStrategy.getIndexStrategy();
        }
    }
    if ((constraints == null) || constraints.isEmpty()) {
        if (targetIndexStrategy != null) {
            // at least use the prefix of a substrategy if chosen
            return new QueryRanges(new byte[][] { targetIndexStrategy.getPrefix() });
        }
        // implies in negative and
        return new QueryRanges();
    // positive infinity
    } else {
        final List<QueryRanges> ranges = new ArrayList<>(constraints.size());
        for (final MultiDimensionalNumericData nd : constraints) {
            ranges.add(indexStrategy.getQueryRanges(nd, maxRanges, hints));
        }
        return ranges.size() > 1 ? new QueryRanges(ranges) : ranges.get(0);
    }
}

Example 4 with HierarchicalNumericIndexStrategy

use of org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy in project geowave by locationtech.

the class RasterDataAdapter method convertToIndex.

@Override
public Iterator<GridCoverage> convertToIndex(final Index index, final GridCoverage gridCoverage) {
    final HierarchicalNumericIndexStrategy indexStrategy = CompoundHierarchicalIndexStrategyWrapper.findHierarchicalStrategy(index.getIndexStrategy());
    if (indexStrategy != null) {
        final CoordinateReferenceSystem sourceCrs = gridCoverage.getCoordinateReferenceSystem();
        final Envelope sampleEnvelope = gridCoverage.getEnvelope();
        final ReferencedEnvelope sampleReferencedEnvelope = new ReferencedEnvelope(new org.locationtech.jts.geom.Envelope(sampleEnvelope.getMinimum(0), sampleEnvelope.getMaximum(0), sampleEnvelope.getMinimum(1), sampleEnvelope.getMaximum(1)), gridCoverage.getCoordinateReferenceSystem());
        ReferencedEnvelope projectedReferenceEnvelope = sampleReferencedEnvelope;
        final CoordinateReferenceSystem indexCrs = GeometryUtils.getIndexCrs(index);
        if (!indexCrs.equals(sourceCrs)) {
            try {
                projectedReferenceEnvelope = sampleReferencedEnvelope.transform(indexCrs, true);
            } catch (TransformException | FactoryException e) {
                LOGGER.warn("Unable to transform envelope of grid coverage to Index CRS", e);
            }
        }
        final MultiDimensionalNumericData bounds;
        if (indexCrs.equals(GeometryUtils.getDefaultCRS())) {
            bounds = IndexUtils.clampAtIndexBounds(GeometryUtils.basicConstraintSetFromEnvelope(projectedReferenceEnvelope).getIndexConstraints(indexStrategy), indexStrategy);
        } else {
            bounds = IndexUtils.clampAtIndexBounds(GeometryUtils.getBoundsFromEnvelope(projectedReferenceEnvelope), indexStrategy);
        }
        final GridEnvelope gridEnvelope = gridCoverage.getGridGeometry().getGridRange();
        // only one set of constraints..hence reference '0' element
        final double[] tileRangePerDimension = new double[bounds.getDimensionCount()];
        final Double[] maxValuesPerDimension = bounds.getMaxValuesPerDimension();
        final Double[] minValuesPerDimension = bounds.getMinValuesPerDimension();
        for (int d = 0; d < tileRangePerDimension.length; d++) {
            tileRangePerDimension[d] = ((maxValuesPerDimension[d] - minValuesPerDimension[d]) * tileSize) / gridEnvelope.getSpan(d);
        }
        final TreeMap<Double, SubStrategy> substrategyMap = new TreeMap<>();
        for (final SubStrategy pyramidLevel : indexStrategy.getSubStrategies()) {
            final double[] idRangePerDimension = pyramidLevel.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
            // to create a pyramid, ingest into each substrategy that is
            // lower resolution than the sample set in at least one
            // dimension and the one substrategy that is at least the same
            // resolution or higher resolution to retain the original
            // resolution as well as possible
            double maxSubstrategyResToSampleSetRes = -Double.MAX_VALUE;
            for (int d = 0; d < tileRangePerDimension.length; d++) {
                final double substrategyResToSampleSetRes = idRangePerDimension[d] / tileRangePerDimension[d];
                maxSubstrategyResToSampleSetRes = Math.max(maxSubstrategyResToSampleSetRes, substrategyResToSampleSetRes);
            }
            substrategyMap.put(maxSubstrategyResToSampleSetRes, pyramidLevel);
        }
        // all entries will be greater than 1 (lower resolution pyramid
        // levels)
        // also try to find the one entry that is closest to 1.0 without
        // going over (this will be the full resolution level)
        // add an epsilon to try to catch any roundoff error
        final double fullRes = 1.0 + MathUtils.EPSILON;
        final Entry<Double, SubStrategy> fullResEntry = substrategyMap.floorEntry(fullRes);
        final List<SubStrategy> pyramidLevels = new ArrayList<>();
        if (fullResEntry != null) {
            pyramidLevels.add(fullResEntry.getValue());
        }
        if (buildPyramid) {
            final NavigableMap<Double, SubStrategy> map = substrategyMap.tailMap(fullRes, false);
            pyramidLevels.addAll(map.values());
        }
        if (pyramidLevels.isEmpty()) {
            // this case shouldn't occur theoretically, but just in case,
            // make sure the substrategy closest to 1.0 is used
            final Entry<Double, SubStrategy> bestEntry = substrategyMap.higherEntry(1.0);
            pyramidLevels.add(bestEntry.getValue());
        }
        return new IteratorWrapper<>(pyramidLevels.iterator(), new MosaicPerPyramidLevelBuilder(bounds, gridCoverage, tileSize, backgroundValuesPerBand, RasterUtils.getFootprint(projectedReferenceEnvelope, gridCoverage), interpolation, projectedReferenceEnvelope.getCoordinateReferenceSystem()));
    }
    LOGGER.warn("Strategy is not an instance of HierarchicalNumericIndexStrategy : " + index.getIndexStrategy().getClass().getName());
    return Collections.<GridCoverage>emptyIterator();
}

Example 5 with HierarchicalNumericIndexStrategy

use of org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy in project geowave by locationtech.

the class DeletePyramidLevelCommand method run.

public void run(final OperationParams params) {
    // Ensure we have all the required arguments
    if (parameters.size() != 1) {
        throw new ParameterException("Requires argument: <store name>");
    }
    final String inputStoreName = parameters.get(0);
    // Attempt to load store.
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, getGeoWaveConfigFile(params), params.getConsole());
    final DataStore store = inputStoreOptions.createDataStore();
    RasterDataAdapter adapter = null;
    for (final DataTypeAdapter<?> type : store.getTypes()) {
        if (isRaster(type) && ((coverageName == null) || coverageName.equals(adapter.getTypeName()))) {
            if (adapter != null) {
                LOGGER.error("Store has multiple coverages.  Must explicitly choose one with --coverage option.");
                return;
            }
            adapter = (RasterDataAdapter) type;
        }
    }
    if (adapter == null) {
        LOGGER.error("Store has no coverages or coverage name not found.");
        return;
    }
    boolean found = false;
    Resolution res = null;
    Index i = null;
    for (final Index index : store.getIndices(adapter.getTypeName())) {
        final HierarchicalNumericIndexStrategy indexStrategy = CompoundHierarchicalIndexStrategyWrapper.findHierarchicalStrategy(index.getIndexStrategy());
        if (indexStrategy != null) {
            for (final SubStrategy s : indexStrategy.getSubStrategies()) {
                if ((s.getPrefix().length == 1) && (s.getPrefix()[0] == level)) {
                    LOGGER.info("Deleting from index " + index.getName());
                    final double[] tileRes = s.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                    final double[] pixelRes = new double[tileRes.length];
                    for (int d = 0; d < tileRes.length; d++) {
                        pixelRes[d] = tileRes[d] / adapter.getTileSize();
                    }
                    found = true;
                    i = index;
                    res = new Resolution(pixelRes);
                    break;
                }
            }
        }
        if (found) {
            break;
        }
    }
    if (!found) {
        LOGGER.error("Store has no indices supporting pyramids.");
        return;
    }
    final byte[][] predefinedSplits = i.getIndexStrategy().getPredefinedSplits();
    // this should account for hash partitioning if used
    final List<ByteArray> partitions = new ArrayList<>();
    if ((predefinedSplits != null) && (predefinedSplits.length > 0)) {
        for (final byte[] split : predefinedSplits) {
            partitions.add(new ByteArray(ArrayUtils.add(split, level.byteValue())));
        }
    } else {
        partitions.add(new ByteArray(new byte[] { level.byteValue() }));
    }
    // delete the resolution from the overview, delete the partitions, and delete the data
    if (inputStoreOptions.getFactoryOptions().getStoreOptions().isPersistDataStatistics()) {
        final DataStatisticsStore statsStore = inputStoreOptions.createDataStatisticsStore();
        boolean overviewStatsFound = false;
        boolean partitionStatsFound = false;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getDataTypeStatistics(adapter, RasterOverviewStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if ((next instanceof RasterOverviewStatistic) && (next.getBinningStrategy() == null)) {
                    final RasterOverviewStatistic statistic = (RasterOverviewStatistic) next;
                    final RasterOverviewValue value = statsStore.getStatisticValue(statistic);
                    if (!value.removeResolution(res)) {
                        LOGGER.error("Unable to remove resolution for pyramid level " + level);
                        return;
                    }
                    statsStore.setStatisticValue(statistic, value);
                    overviewStatsFound = true;
                }
            }
        }
        if (!overviewStatsFound) {
            LOGGER.error("Unable to find overview stats for coverage " + adapter.getTypeName());
            return;
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getIndexStatistics(i, PartitionsStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if (next instanceof PartitionsStatistic) {
                    if ((next.getBinningStrategy() != null) && (next.getBinningStrategy() instanceof DataTypeBinningStrategy)) {
                        final PartitionsStatistic statistic = (PartitionsStatistic) next;
                        final PartitionsValue value = statsStore.getStatisticValue((PartitionsStatistic) next, DataTypeBinningStrategy.getBin(adapter));
                        for (final ByteArray p : partitions) {
                            if (!value.getValue().remove(p)) {
                                LOGGER.error("Unable to remove partition " + p.getHexString() + " for pyramid level " + level);
                                return;
                            }
                        }
                        statsStore.setStatisticValue(statistic, value, DataTypeBinningStrategy.getBin(adapter));
                        partitionStatsFound = true;
                    }
                }
            }
        }
        if (!partitionStatsFound) {
            LOGGER.error("Unable to find partition stats for coverage " + adapter.getTypeName() + " and index " + i.getName());
            return;
        }
    }
    for (final ByteArray p : partitions) {
        store.delete(QueryBuilder.newBuilder().constraints(QueryBuilder.newBuilder().constraintsFactory().prefix(p.getBytes(), null)).addTypeName(adapter.getTypeName()).indexName(i.getName()).build());
    }
}