use of org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy in project geowave by locationtech.
the class IndexImpl method getDefaultStatistics.
@Override
public List<Statistic<? extends StatisticValue<?>>> getDefaultStatistics() {
List<Statistic<? extends StatisticValue<?>>> statistics = Lists.newArrayListWithCapacity(6);
IndexMetaDataSetStatistic metadata = new IndexMetaDataSetStatistic(getName(), indexStrategy.createMetaData());
metadata.setBinningStrategy(new DataTypeBinningStrategy());
metadata.setInternal();
statistics.add(metadata);
DuplicateEntryCountStatistic duplicateCounts = new DuplicateEntryCountStatistic(getName());
duplicateCounts.setBinningStrategy(new DataTypeBinningStrategy());
duplicateCounts.setInternal();
statistics.add(duplicateCounts);
PartitionsStatistic partitions = new PartitionsStatistic(getName());
partitions.setBinningStrategy(new DataTypeBinningStrategy());
partitions.setInternal();
statistics.add(partitions);
DifferingVisibilityCountStatistic differingFieldVisibility = new DifferingVisibilityCountStatistic(getName());
differingFieldVisibility.setBinningStrategy(new DataTypeBinningStrategy());
differingFieldVisibility.setInternal();
statistics.add(differingFieldVisibility);
FieldVisibilityCountStatistic fieldVisibilityCount = new FieldVisibilityCountStatistic(getName());
fieldVisibilityCount.setBinningStrategy(new DataTypeBinningStrategy());
fieldVisibilityCount.setInternal();
statistics.add(fieldVisibilityCount);
RowRangeHistogramStatistic rowRangeHistogram = new RowRangeHistogramStatistic(getName());
rowRangeHistogram.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
rowRangeHistogram.setInternal();
statistics.add(rowRangeHistogram);
return statistics;
}
use of org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy in project geowave by locationtech.
the class ListStatsCommand method performStatsCommand.
@Override
protected boolean performStatsCommand(final DataStorePluginOptions storeOptions, final StatsCommandLineOptions statsOptions, final Console console) throws IOException {
final DataStatisticsStore statsStore = storeOptions.createDataStatisticsStore();
final IndexStore indexStore = storeOptions.createIndexStore();
final String[] authorizations = getAuthorizations(statsOptions.getAuthorizations());
DataTypeAdapter<?> adapter = null;
if (statsOptions.getTypeName() != null) {
adapter = storeOptions.createDataStore().getType(statsOptions.getTypeName());
if (adapter == null) {
throw new ParameterException("A type called " + statsOptions.getTypeName() + " was not found.");
}
}
StatisticType<StatisticValue<Object>> statisticType = null;
if (statsOptions.getStatType() != null) {
statisticType = StatisticsRegistry.instance().getStatisticType(statsOptions.getStatType());
if (statisticType == null) {
throw new ParameterException("Unrecognized statistic type: " + statsOptions.getStatType());
}
}
List<String> headers = null;
List<Statistic<?>> statsToList = Lists.newLinkedList();
ValueTransformer transformer = null;
Predicate<StatisticValue<?>> filter;
if (statsOptions.getIndexName() != null) {
if (statisticType != null && !(statisticType instanceof IndexStatisticType)) {
throw new ParameterException("Only index statistic types can be specified when listing statistics for a specific index.");
}
Index index = indexStore.getIndex(statsOptions.getIndexName());
if (index == null) {
throw new ParameterException("An index called " + statsOptions.getIndexName() + " was not found.");
}
headers = Lists.newArrayList("Statistic", "Tag", "Bin", "Value");
transformer = new ValueToRow();
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getIndexStatistics(index, statisticType, statsOptions.getTag())) {
if (adapter != null) {
stats.forEachRemaining(stat -> {
if (stat.getBinningStrategy() instanceof DataTypeBinningStrategy || (stat.getBinningStrategy() instanceof CompositeBinningStrategy && ((CompositeBinningStrategy) stat.getBinningStrategy()).usesStrategy(DataTypeBinningStrategy.class))) {
statsToList.add(stat);
}
});
filter = new IndexAdapterFilter(adapter.getTypeName());
} else {
stats.forEachRemaining(statsToList::add);
filter = null;
}
}
} else if (statsOptions.getTypeName() != null) {
filter = null;
if (statsOptions.getFieldName() != null) {
if (statisticType != null && !(statisticType instanceof FieldStatisticType)) {
throw new ParameterException("Only field statistic types can be specified when listing statistics for a specific field.");
}
headers = Lists.newArrayList("Statistic", "Tag", "Bin", "Value");
transformer = new ValueToRow();
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getFieldStatistics(adapter, statisticType, statsOptions.getFieldName(), statsOptions.getTag())) {
stats.forEachRemaining(statsToList::add);
}
} else {
if (statisticType != null && statisticType instanceof IndexStatisticType) {
throw new ParameterException("Only data type and field statistic types can be specified when listing statistics for a specific data type.");
}
headers = Lists.newArrayList("Statistic", "Tag", "Field", "Bin", "Value");
transformer = new ValueToFieldRow();
if (statisticType == null || statisticType instanceof DataTypeStatisticType) {
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getDataTypeStatistics(adapter, statisticType, statsOptions.getTag())) {
stats.forEachRemaining(statsToList::add);
}
}
if (statisticType == null || statisticType instanceof FieldStatisticType) {
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getFieldStatistics(adapter, statisticType, null, statsOptions.getTag())) {
stats.forEachRemaining(statsToList::add);
}
}
}
} else if (statsOptions.getFieldName() != null) {
throw new ParameterException("A type name must be supplied with a field name.");
} else {
filter = null;
headers = Lists.newArrayList("Index/Adapter", "Statistic", "Tag", "Field", "Bin", "Value");
transformer = new ValueToAllRow();
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getAllStatistics(statisticType)) {
stats.forEachRemaining(stat -> {
if (statsOptions.getTag() == null || stat.getTag().equals(statsOptions.getTag())) {
statsToList.add(stat);
}
});
}
}
Collections.sort(statsToList, new StatComparator());
try (StatisticsValueIterator values = new StatisticsValueIterator(statsStore, statsToList.iterator(), null, authorizations)) {
Iterator<List<Object>> rows = Iterators.transform(filter == null ? values : Iterators.filter(values, v -> filter.test(v)), transformer::transform);
if (limit != null) {
rows = Iterators.limit(rows, limit);
}
if (rows.hasNext()) {
if (csv) {
StringBuilder sb = new StringBuilder();
sb.append(Arrays.toString(headers.toArray()));
rows.forEachRemaining(row -> sb.append(Arrays.toString(row.toArray())));
retValue = sb.toString();
console.println(retValue);
} else {
console.println("Matching statistics:");
ConsoleTablePrinter printer = new ConsoleTablePrinter(0, limit != null ? limit : 30, console);
printer.print(headers, rows);
}
} else {
console.println("No matching statistics were found.");
}
}
return true;
}
use of org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy in project geowave by locationtech.
the class ChooseBestMatchIndexQueryStrategy method getIndices.
@Override
public CloseableIterator<Index> getIndices(final DataStatisticsStore statisticsStore, final AdapterIndexMappingStore mappingStore, final QueryConstraints query, final Index[] indices, final InternalDataAdapter<?> adapter, final Map<QueryHint, Object> hints) {
return new CloseableIterator<Index>() {
Index nextIdx = null;
boolean done = false;
int i = 0;
@Override
public boolean hasNext() {
long min = Long.MAX_VALUE;
Index bestIdx = null;
while (!done && (i < indices.length)) {
nextIdx = indices[i++];
if (nextIdx.getIndexStrategy().getOrderedDimensionDefinitions().length == 0) {
continue;
}
final List<MultiDimensionalNumericData> constraints = query.getIndexConstraints(nextIdx);
RowRangeHistogramStatistic rowRangeHistogramStatistic = null;
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statisticsStore.getIndexStatistics(nextIdx, RowRangeHistogramStatistic.STATS_TYPE, Statistic.INTERNAL_TAG)) {
if (stats.hasNext()) {
final Statistic<?> statistic = stats.next();
if ((statistic instanceof RowRangeHistogramStatistic) && (statistic.getBinningStrategy() instanceof CompositeBinningStrategy) && ((CompositeBinningStrategy) statistic.getBinningStrategy()).isOfType(DataTypeBinningStrategy.class, PartitionBinningStrategy.class)) {
rowRangeHistogramStatistic = (RowRangeHistogramStatistic) statistic;
}
}
}
if (rowRangeHistogramStatistic == null) {
LOGGER.warn("Best Match Heuristic requires statistic RowRangeHistogramStatistics for each index to properly choose an index.");
}
if (IndexUtils.isFullTableScan(constraints)) {
// result in a full table scan
if (bestIdx == null) {
bestIdx = nextIdx;
}
} else {
final int maxRangeDecomposition;
if (hints.containsKey(QueryHint.MAX_RANGE_DECOMPOSITION)) {
maxRangeDecomposition = (Integer) hints.get(QueryHint.MAX_RANGE_DECOMPOSITION);
} else {
LOGGER.warn("No max range decomposition hint was provided, this should be provided from the data store options");
maxRangeDecomposition = 2000;
}
final QueryRanges ranges = DataStoreUtils.constraintsToQueryRanges(constraints, nextIdx, null, maxRangeDecomposition);
final long temp = DataStoreUtils.cardinality(statisticsStore, rowRangeHistogramStatistic, adapter, nextIdx, ranges);
if (temp < min) {
bestIdx = nextIdx;
min = temp;
}
}
}
nextIdx = bestIdx;
done = true;
return nextIdx != null;
}
@Override
public Index next() throws NoSuchElementException {
if (nextIdx == null) {
throw new NoSuchElementException();
}
final Index returnVal = nextIdx;
nextIdx = null;
return returnVal;
}
@Override
public void remove() {
}
@Override
public void close() {
}
};
}
use of org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy in project geowave by locationtech.
the class ChooseBestMatchIndexQueryStrategyTest method testChooseSpatialTemporalWithStats.
@Test
public void testChooseSpatialTemporalWithStats() {
final Index temporalindex = new SpatialTemporalIndexBuilder().createIndex();
final Index spatialIndex = new SpatialIndexBuilder().createIndex();
final RowRangeHistogramStatistic rangeTempStats = new RowRangeHistogramStatistic(temporalindex.getName());
rangeTempStats.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
rangeTempStats.setInternal();
final RowRangeHistogramStatistic rangeStats = new RowRangeHistogramStatistic(spatialIndex.getName());
rangeStats.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
rangeStats.setInternal();
final Map<StatisticId<?>, Map<ByteArray, StatisticValue<?>>> statsMap = new HashMap<>();
final ChooseBestMatchIndexQueryStrategy strategy = new ChooseBestMatchIndexQueryStrategy();
final ConstraintSet cs1 = new ConstraintSet();
cs1.addConstraint(LatitudeDefinition.class, new ConstraintData(new ConstrainedIndexValue(0.3, 0.5), true));
cs1.addConstraint(LongitudeDefinition.class, new ConstraintData(new ConstrainedIndexValue(0.4, 0.7), true));
final ConstraintSet cs2a = new ConstraintSet();
cs2a.addConstraint(TimeDefinition.class, new ConstraintData(new ConstrainedIndexValue(0.1, 0.2), true));
final ConstraintsByClass constraints = new ConstraintsByClass(Arrays.asList(cs2a)).merge(Collections.singletonList(cs1));
final BasicQueryByClass query = new BasicQueryByClass(constraints);
final NumericIndexStrategy temporalIndexStrategy = new SpatialTemporalIndexBuilder().createIndex().getIndexStrategy();
final Random r = new Random(SEED);
for (int i = 0; i < ROWS; i++) {
final double x = r.nextDouble();
final double y = r.nextDouble();
final double t = r.nextDouble();
final InsertionIds id = temporalIndexStrategy.getInsertionIds(new BasicNumericDataset(new NumericData[] { new NumericValue(x), new NumericValue(y), new NumericValue(t) }));
for (final SinglePartitionInsertionIds range : id.getPartitionKeys()) {
Map<ByteArray, StatisticValue<?>> binValues = statsMap.get(rangeTempStats.getId());
if (binValues == null) {
binValues = Maps.newHashMap();
statsMap.put(rangeTempStats.getId(), binValues);
}
final ByteArray bin = CompositeBinningStrategy.getBin(DataTypeBinningStrategy.getBin((String) null), PartitionBinningStrategy.getBin(range.getPartitionKey()));
RowRangeHistogramValue value = (RowRangeHistogramValue) binValues.get(bin);
if (value == null) {
value = rangeTempStats.createEmpty();
value.setBin(bin);
binValues.put(bin, value);
}
((StatisticsIngestCallback) value).entryIngested(null, null, new GeoWaveRowImpl(new GeoWaveKeyImpl(new byte[] { 1 }, (short) 1, range.getPartitionKey(), range.getSortKeys().get(0), 0), new GeoWaveValue[] {}));
}
}
final Index index = new SpatialIndexBuilder().createIndex();
final NumericIndexStrategy indexStrategy = index.getIndexStrategy();
for (int i = 0; i < ROWS; i++) {
final double x = r.nextDouble();
final double y = r.nextDouble();
final double t = r.nextDouble();
final InsertionIds id = indexStrategy.getInsertionIds(new BasicNumericDataset(new NumericData[] { new NumericValue(x), new NumericValue(y), new NumericValue(t) }));
for (final SinglePartitionInsertionIds range : id.getPartitionKeys()) {
Map<ByteArray, StatisticValue<?>> binValues = statsMap.get(rangeStats.getId());
if (binValues == null) {
binValues = Maps.newHashMap();
statsMap.put(rangeStats.getId(), binValues);
}
final ByteArray bin = CompositeBinningStrategy.getBin(DataTypeBinningStrategy.getBin((String) null), PartitionBinningStrategy.getBin(range.getPartitionKey()));
RowRangeHistogramValue value = (RowRangeHistogramValue) binValues.get(bin);
if (value == null) {
value = rangeStats.createEmpty();
value.setBin(bin);
binValues.put(bin, value);
}
((StatisticsIngestCallback) value).entryIngested(null, null, new GeoWaveRowImpl(new GeoWaveKeyImpl(new byte[] { 1 }, (short) 1, range.getPartitionKey(), range.getSortKeys().get(0), 0), new GeoWaveValue[] {}));
}
}
final Iterator<Index> it = getIndices(new TestDataStatisticsStore(Lists.newArrayList(rangeStats, rangeTempStats), statsMap), query, strategy);
assertTrue(it.hasNext());
assertEquals(temporalindex.getName(), it.next().getName());
assertFalse(it.hasNext());
}
use of org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy in project geowave by locationtech.
the class DataStatisticsStoreImpl method removeTypeSpecificStatisticValues.
@SuppressWarnings("unchecked")
@Override
public boolean removeTypeSpecificStatisticValues(final IndexStatistic<?> indexStatistic, final String typeName) {
if (indexStatistic.getBinningStrategy() == null) {
return false;
}
final ByteArray adapterBin = DataTypeBinningStrategy.getBin(typeName);
boolean removed = false;
if (indexStatistic.getBinningStrategy() instanceof DataTypeBinningStrategy) {
removed = removeStatisticValue(indexStatistic, adapterBin);
} else if ((indexStatistic.getBinningStrategy() instanceof CompositeBinningStrategy) && ((CompositeBinningStrategy) indexStatistic.getBinningStrategy()).usesStrategy(DataTypeBinningStrategy.class)) {
final CompositeBinningStrategy binningStrategy = (CompositeBinningStrategy) indexStatistic.getBinningStrategy();
// TODO: The current metadata deleter only deletes exact values. One future optimization
// could be to allow it to delete with a primary Id prefix. If the strategy index is 0,
// a prefix delete could be used.
final List<ByteArray> binsToRemove = Lists.newLinkedList();
try (CloseableIterator<StatisticValue<Object>> valueIter = getStatisticValues((Statistic<StatisticValue<Object>>) indexStatistic)) {
while (valueIter.hasNext()) {
final ByteArray bin = valueIter.next().getBin();
if (binningStrategy.binMatches(DataTypeBinningStrategy.class, bin, adapterBin)) {
binsToRemove.add(bin);
}
}
}
for (final ByteArray bin : binsToRemove) {
removed = removeStatisticValue(indexStatistic, bin) || removed;
}
}
return removed;
}
Aggregations