Examples with DataTypeBinningStrategy org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy used on opensource projects

Example 1 with DataTypeBinningStrategy

use of org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy in project geowave by locationtech.

the class IndexImpl method getDefaultStatistics.

@Override
public List<Statistic<? extends StatisticValue<?>>> getDefaultStatistics() {
    List<Statistic<? extends StatisticValue<?>>> statistics = Lists.newArrayListWithCapacity(6);
    IndexMetaDataSetStatistic metadata = new IndexMetaDataSetStatistic(getName(), indexStrategy.createMetaData());
    metadata.setBinningStrategy(new DataTypeBinningStrategy());
    metadata.setInternal();
    statistics.add(metadata);
    DuplicateEntryCountStatistic duplicateCounts = new DuplicateEntryCountStatistic(getName());
    duplicateCounts.setBinningStrategy(new DataTypeBinningStrategy());
    duplicateCounts.setInternal();
    statistics.add(duplicateCounts);
    PartitionsStatistic partitions = new PartitionsStatistic(getName());
    partitions.setBinningStrategy(new DataTypeBinningStrategy());
    partitions.setInternal();
    statistics.add(partitions);
    DifferingVisibilityCountStatistic differingFieldVisibility = new DifferingVisibilityCountStatistic(getName());
    differingFieldVisibility.setBinningStrategy(new DataTypeBinningStrategy());
    differingFieldVisibility.setInternal();
    statistics.add(differingFieldVisibility);
    FieldVisibilityCountStatistic fieldVisibilityCount = new FieldVisibilityCountStatistic(getName());
    fieldVisibilityCount.setBinningStrategy(new DataTypeBinningStrategy());
    fieldVisibilityCount.setInternal();
    statistics.add(fieldVisibilityCount);
    RowRangeHistogramStatistic rowRangeHistogram = new RowRangeHistogramStatistic(getName());
    rowRangeHistogram.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
    rowRangeHistogram.setInternal();
    statistics.add(rowRangeHistogram);
    return statistics;
}

Example 2 with DataTypeBinningStrategy

use of org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy in project geowave by locationtech.

the class ListStatsCommand method performStatsCommand.

@Override
protected boolean performStatsCommand(final DataStorePluginOptions storeOptions, final StatsCommandLineOptions statsOptions, final Console console) throws IOException {
    final DataStatisticsStore statsStore = storeOptions.createDataStatisticsStore();
    final IndexStore indexStore = storeOptions.createIndexStore();
    final String[] authorizations = getAuthorizations(statsOptions.getAuthorizations());
    DataTypeAdapter<?> adapter = null;
    if (statsOptions.getTypeName() != null) {
        adapter = storeOptions.createDataStore().getType(statsOptions.getTypeName());
        if (adapter == null) {
            throw new ParameterException("A type called " + statsOptions.getTypeName() + " was not found.");
        }
    }
    StatisticType<StatisticValue<Object>> statisticType = null;
    if (statsOptions.getStatType() != null) {
        statisticType = StatisticsRegistry.instance().getStatisticType(statsOptions.getStatType());
        if (statisticType == null) {
            throw new ParameterException("Unrecognized statistic type: " + statsOptions.getStatType());
        }
    }
    List<String> headers = null;
    List<Statistic<?>> statsToList = Lists.newLinkedList();
    ValueTransformer transformer = null;
    Predicate<StatisticValue<?>> filter;
    if (statsOptions.getIndexName() != null) {
        if (statisticType != null && !(statisticType instanceof IndexStatisticType)) {
            throw new ParameterException("Only index statistic types can be specified when listing statistics for a specific index.");
        }
        Index index = indexStore.getIndex(statsOptions.getIndexName());
        if (index == null) {
            throw new ParameterException("An index called " + statsOptions.getIndexName() + " was not found.");
        }
        headers = Lists.newArrayList("Statistic", "Tag", "Bin", "Value");
        transformer = new ValueToRow();
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getIndexStatistics(index, statisticType, statsOptions.getTag())) {
            if (adapter != null) {
                stats.forEachRemaining(stat -> {
                    if (stat.getBinningStrategy() instanceof DataTypeBinningStrategy || (stat.getBinningStrategy() instanceof CompositeBinningStrategy && ((CompositeBinningStrategy) stat.getBinningStrategy()).usesStrategy(DataTypeBinningStrategy.class))) {
                        statsToList.add(stat);
                    }
                });
                filter = new IndexAdapterFilter(adapter.getTypeName());
            } else {
                stats.forEachRemaining(statsToList::add);
                filter = null;
            }
        }
    } else if (statsOptions.getTypeName() != null) {
        filter = null;
        if (statsOptions.getFieldName() != null) {
            if (statisticType != null && !(statisticType instanceof FieldStatisticType)) {
                throw new ParameterException("Only field statistic types can be specified when listing statistics for a specific field.");
            }
            headers = Lists.newArrayList("Statistic", "Tag", "Bin", "Value");
            transformer = new ValueToRow();
            try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getFieldStatistics(adapter, statisticType, statsOptions.getFieldName(), statsOptions.getTag())) {
                stats.forEachRemaining(statsToList::add);
            }
        } else {
            if (statisticType != null && statisticType instanceof IndexStatisticType) {
                throw new ParameterException("Only data type and field statistic types can be specified when listing statistics for a specific data type.");
            }
            headers = Lists.newArrayList("Statistic", "Tag", "Field", "Bin", "Value");
            transformer = new ValueToFieldRow();
            if (statisticType == null || statisticType instanceof DataTypeStatisticType) {
                try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getDataTypeStatistics(adapter, statisticType, statsOptions.getTag())) {
                    stats.forEachRemaining(statsToList::add);
                }
            }
            if (statisticType == null || statisticType instanceof FieldStatisticType) {
                try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getFieldStatistics(adapter, statisticType, null, statsOptions.getTag())) {
                    stats.forEachRemaining(statsToList::add);
                }
            }
        }
    } else if (statsOptions.getFieldName() != null) {
        throw new ParameterException("A type name must be supplied with a field name.");
    } else {
        filter = null;
        headers = Lists.newArrayList("Index/Adapter", "Statistic", "Tag", "Field", "Bin", "Value");
        transformer = new ValueToAllRow();
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getAllStatistics(statisticType)) {
            stats.forEachRemaining(stat -> {
                if (statsOptions.getTag() == null || stat.getTag().equals(statsOptions.getTag())) {
                    statsToList.add(stat);
                }
            });
        }
    }
    Collections.sort(statsToList, new StatComparator());
    try (StatisticsValueIterator values = new StatisticsValueIterator(statsStore, statsToList.iterator(), null, authorizations)) {
        Iterator<List<Object>> rows = Iterators.transform(filter == null ? values : Iterators.filter(values, v -> filter.test(v)), transformer::transform);
        if (limit != null) {
            rows = Iterators.limit(rows, limit);
        }
        if (rows.hasNext()) {
            if (csv) {
                StringBuilder sb = new StringBuilder();
                sb.append(Arrays.toString(headers.toArray()));
                rows.forEachRemaining(row -> sb.append(Arrays.toString(row.toArray())));
                retValue = sb.toString();
                console.println(retValue);
            } else {
                console.println("Matching statistics:");
                ConsoleTablePrinter printer = new ConsoleTablePrinter(0, limit != null ? limit : 30, console);
                printer.print(headers, rows);
            }
        } else {
            console.println("No matching statistics were found.");
        }
    }
    return true;
}

Example 3 with DataTypeBinningStrategy

use of org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy in project geowave by locationtech.

the class DataStatisticsStoreImpl method removeTypeSpecificStatisticValues.

@SuppressWarnings("unchecked")
@Override
public boolean removeTypeSpecificStatisticValues(final IndexStatistic<?> indexStatistic, final String typeName) {
    if (indexStatistic.getBinningStrategy() == null) {
        return false;
    }
    final ByteArray adapterBin = DataTypeBinningStrategy.getBin(typeName);
    boolean removed = false;
    if (indexStatistic.getBinningStrategy() instanceof DataTypeBinningStrategy) {
        removed = removeStatisticValue(indexStatistic, adapterBin);
    } else if ((indexStatistic.getBinningStrategy() instanceof CompositeBinningStrategy) && ((CompositeBinningStrategy) indexStatistic.getBinningStrategy()).usesStrategy(DataTypeBinningStrategy.class)) {
        final CompositeBinningStrategy binningStrategy = (CompositeBinningStrategy) indexStatistic.getBinningStrategy();
        // TODO: The current metadata deleter only deletes exact values. One future optimization
        // could be to allow it to delete with a primary Id prefix. If the strategy index is 0,
        // a prefix delete could be used.
        final List<ByteArray> binsToRemove = Lists.newLinkedList();
        try (CloseableIterator<StatisticValue<Object>> valueIter = getStatisticValues((Statistic<StatisticValue<Object>>) indexStatistic)) {
            while (valueIter.hasNext()) {
                final ByteArray bin = valueIter.next().getBin();
                if (binningStrategy.binMatches(DataTypeBinningStrategy.class, bin, adapterBin)) {
                    binsToRemove.add(bin);
                }
            }
        }
        for (final ByteArray bin : binsToRemove) {
            removed = removeStatisticValue(indexStatistic, bin) || removed;
        }
    }
    return removed;
}

Example 4 with DataTypeBinningStrategy

use of org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy in project geowave by locationtech.

the class DeletePyramidLevelCommand method run.

public void run(final OperationParams params) {
    // Ensure we have all the required arguments
    if (parameters.size() != 1) {
        throw new ParameterException("Requires argument: <store name>");
    }
    final String inputStoreName = parameters.get(0);
    // Attempt to load store.
    inputStoreOptions = CLIUtils.loadStore(inputStoreName, getGeoWaveConfigFile(params), params.getConsole());
    final DataStore store = inputStoreOptions.createDataStore();
    RasterDataAdapter adapter = null;
    for (final DataTypeAdapter<?> type : store.getTypes()) {
        if (isRaster(type) && ((coverageName == null) || coverageName.equals(adapter.getTypeName()))) {
            if (adapter != null) {
                LOGGER.error("Store has multiple coverages.  Must explicitly choose one with --coverage option.");
                return;
            }
            adapter = (RasterDataAdapter) type;
        }
    }
    if (adapter == null) {
        LOGGER.error("Store has no coverages or coverage name not found.");
        return;
    }
    boolean found = false;
    Resolution res = null;
    Index i = null;
    for (final Index index : store.getIndices(adapter.getTypeName())) {
        final HierarchicalNumericIndexStrategy indexStrategy = CompoundHierarchicalIndexStrategyWrapper.findHierarchicalStrategy(index.getIndexStrategy());
        if (indexStrategy != null) {
            for (final SubStrategy s : indexStrategy.getSubStrategies()) {
                if ((s.getPrefix().length == 1) && (s.getPrefix()[0] == level)) {
                    LOGGER.info("Deleting from index " + index.getName());
                    final double[] tileRes = s.getIndexStrategy().getHighestPrecisionIdRangePerDimension();
                    final double[] pixelRes = new double[tileRes.length];
                    for (int d = 0; d < tileRes.length; d++) {
                        pixelRes[d] = tileRes[d] / adapter.getTileSize();
                    }
                    found = true;
                    i = index;
                    res = new Resolution(pixelRes);
                    break;
                }
            }
        }
        if (found) {
            break;
        }
    }
    if (!found) {
        LOGGER.error("Store has no indices supporting pyramids.");
        return;
    }
    final byte[][] predefinedSplits = i.getIndexStrategy().getPredefinedSplits();
    // this should account for hash partitioning if used
    final List<ByteArray> partitions = new ArrayList<>();
    if ((predefinedSplits != null) && (predefinedSplits.length > 0)) {
        for (final byte[] split : predefinedSplits) {
            partitions.add(new ByteArray(ArrayUtils.add(split, level.byteValue())));
        }
    } else {
        partitions.add(new ByteArray(new byte[] { level.byteValue() }));
    }
    // delete the resolution from the overview, delete the partitions, and delete the data
    if (inputStoreOptions.getFactoryOptions().getStoreOptions().isPersistDataStatistics()) {
        final DataStatisticsStore statsStore = inputStoreOptions.createDataStatisticsStore();
        boolean overviewStatsFound = false;
        boolean partitionStatsFound = false;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getDataTypeStatistics(adapter, RasterOverviewStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if ((next instanceof RasterOverviewStatistic) && (next.getBinningStrategy() == null)) {
                    final RasterOverviewStatistic statistic = (RasterOverviewStatistic) next;
                    final RasterOverviewValue value = statsStore.getStatisticValue(statistic);
                    if (!value.removeResolution(res)) {
                        LOGGER.error("Unable to remove resolution for pyramid level " + level);
                        return;
                    }
                    statsStore.setStatisticValue(statistic, value);
                    overviewStatsFound = true;
                }
            }
        }
        if (!overviewStatsFound) {
            LOGGER.error("Unable to find overview stats for coverage " + adapter.getTypeName());
            return;
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> it = statsStore.getIndexStatistics(i, PartitionsStatistic.STATS_TYPE, null)) {
            while (it.hasNext()) {
                final Statistic<? extends StatisticValue<?>> next = it.next();
                if (next instanceof PartitionsStatistic) {
                    if ((next.getBinningStrategy() != null) && (next.getBinningStrategy() instanceof DataTypeBinningStrategy)) {
                        final PartitionsStatistic statistic = (PartitionsStatistic) next;
                        final PartitionsValue value = statsStore.getStatisticValue((PartitionsStatistic) next, DataTypeBinningStrategy.getBin(adapter));
                        for (final ByteArray p : partitions) {
                            if (!value.getValue().remove(p)) {
                                LOGGER.error("Unable to remove partition " + p.getHexString() + " for pyramid level " + level);
                                return;
                            }
                        }
                        statsStore.setStatisticValue(statistic, value, DataTypeBinningStrategy.getBin(adapter));
                        partitionStatsFound = true;
                    }
                }
            }
        }
        if (!partitionStatsFound) {
            LOGGER.error("Unable to find partition stats for coverage " + adapter.getTypeName() + " and index " + i.getName());
            return;
        }
    }
    for (final ByteArray p : partitions) {
        store.delete(QueryBuilder.newBuilder().constraints(QueryBuilder.newBuilder().constraintsFactory().prefix(p.getBytes(), null)).addTypeName(adapter.getTypeName()).indexName(i.getName()).build());
    }
}

Example 5 with DataTypeBinningStrategy

use of org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy in project geowave by locationtech.

the class ChooseBestMatchIndexQueryStrategyTest method testChooseSpatialTemporalWithStats.

@Test
public void testChooseSpatialTemporalWithStats() {
    final Index temporalindex = new SpatialTemporalIndexBuilder().createIndex();
    final Index spatialIndex = new SpatialIndexBuilder().createIndex();
    final RowRangeHistogramStatistic rangeTempStats = new RowRangeHistogramStatistic(temporalindex.getName());
    rangeTempStats.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
    rangeTempStats.setInternal();
    final RowRangeHistogramStatistic rangeStats = new RowRangeHistogramStatistic(spatialIndex.getName());
    rangeStats.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
    rangeStats.setInternal();
    final Map<StatisticId<?>, Map<ByteArray, StatisticValue<?>>> statsMap = new HashMap<>();
    final ChooseBestMatchIndexQueryStrategy strategy = new ChooseBestMatchIndexQueryStrategy();
    final ConstraintSet cs1 = new ConstraintSet();
    cs1.addConstraint(LatitudeDefinition.class, new ConstraintData(new ConstrainedIndexValue(0.3, 0.5), true));
    cs1.addConstraint(LongitudeDefinition.class, new ConstraintData(new ConstrainedIndexValue(0.4, 0.7), true));
    final ConstraintSet cs2a = new ConstraintSet();
    cs2a.addConstraint(TimeDefinition.class, new ConstraintData(new ConstrainedIndexValue(0.1, 0.2), true));
    final ConstraintsByClass constraints = new ConstraintsByClass(Arrays.asList(cs2a)).merge(Collections.singletonList(cs1));
    final BasicQueryByClass query = new BasicQueryByClass(constraints);
    final NumericIndexStrategy temporalIndexStrategy = new SpatialTemporalIndexBuilder().createIndex().getIndexStrategy();
    final Random r = new Random(SEED);
    for (int i = 0; i < ROWS; i++) {
        final double x = r.nextDouble();
        final double y = r.nextDouble();
        final double t = r.nextDouble();
        final InsertionIds id = temporalIndexStrategy.getInsertionIds(new BasicNumericDataset(new NumericData[] { new NumericValue(x), new NumericValue(y), new NumericValue(t) }));
        for (final SinglePartitionInsertionIds range : id.getPartitionKeys()) {
            Map<ByteArray, StatisticValue<?>> binValues = statsMap.get(rangeTempStats.getId());
            if (binValues == null) {
                binValues = Maps.newHashMap();
                statsMap.put(rangeTempStats.getId(), binValues);
            }
            final ByteArray bin = CompositeBinningStrategy.getBin(DataTypeBinningStrategy.getBin((String) null), PartitionBinningStrategy.getBin(range.getPartitionKey()));
            RowRangeHistogramValue value = (RowRangeHistogramValue) binValues.get(bin);
            if (value == null) {
                value = rangeTempStats.createEmpty();
                value.setBin(bin);
                binValues.put(bin, value);
            }
            ((StatisticsIngestCallback) value).entryIngested(null, null, new GeoWaveRowImpl(new GeoWaveKeyImpl(new byte[] { 1 }, (short) 1, range.getPartitionKey(), range.getSortKeys().get(0), 0), new GeoWaveValue[] {}));
        }
    }
    final Index index = new SpatialIndexBuilder().createIndex();
    final NumericIndexStrategy indexStrategy = index.getIndexStrategy();
    for (int i = 0; i < ROWS; i++) {
        final double x = r.nextDouble();
        final double y = r.nextDouble();
        final double t = r.nextDouble();
        final InsertionIds id = indexStrategy.getInsertionIds(new BasicNumericDataset(new NumericData[] { new NumericValue(x), new NumericValue(y), new NumericValue(t) }));
        for (final SinglePartitionInsertionIds range : id.getPartitionKeys()) {
            Map<ByteArray, StatisticValue<?>> binValues = statsMap.get(rangeStats.getId());
            if (binValues == null) {
                binValues = Maps.newHashMap();
                statsMap.put(rangeStats.getId(), binValues);
            }
            final ByteArray bin = CompositeBinningStrategy.getBin(DataTypeBinningStrategy.getBin((String) null), PartitionBinningStrategy.getBin(range.getPartitionKey()));
            RowRangeHistogramValue value = (RowRangeHistogramValue) binValues.get(bin);
            if (value == null) {
                value = rangeStats.createEmpty();
                value.setBin(bin);
                binValues.put(bin, value);
            }
            ((StatisticsIngestCallback) value).entryIngested(null, null, new GeoWaveRowImpl(new GeoWaveKeyImpl(new byte[] { 1 }, (short) 1, range.getPartitionKey(), range.getSortKeys().get(0), 0), new GeoWaveValue[] {}));
        }
    }
    final Iterator<Index> it = getIndices(new TestDataStatisticsStore(Lists.newArrayList(rangeStats, rangeTempStats), statsMap), query, strategy);
    assertTrue(it.hasNext());
    assertEquals(temporalindex.getName(), it.next().getName());
    assertFalse(it.hasNext());
}