use of org.locationtech.geowave.core.store.api.StatisticValue in project geowave by locationtech.
the class IndexImpl method getDefaultStatistics.
@Override
public List<Statistic<? extends StatisticValue<?>>> getDefaultStatistics() {
List<Statistic<? extends StatisticValue<?>>> statistics = Lists.newArrayListWithCapacity(6);
IndexMetaDataSetStatistic metadata = new IndexMetaDataSetStatistic(getName(), indexStrategy.createMetaData());
metadata.setBinningStrategy(new DataTypeBinningStrategy());
metadata.setInternal();
statistics.add(metadata);
DuplicateEntryCountStatistic duplicateCounts = new DuplicateEntryCountStatistic(getName());
duplicateCounts.setBinningStrategy(new DataTypeBinningStrategy());
duplicateCounts.setInternal();
statistics.add(duplicateCounts);
PartitionsStatistic partitions = new PartitionsStatistic(getName());
partitions.setBinningStrategy(new DataTypeBinningStrategy());
partitions.setInternal();
statistics.add(partitions);
DifferingVisibilityCountStatistic differingFieldVisibility = new DifferingVisibilityCountStatistic(getName());
differingFieldVisibility.setBinningStrategy(new DataTypeBinningStrategy());
differingFieldVisibility.setInternal();
statistics.add(differingFieldVisibility);
FieldVisibilityCountStatistic fieldVisibilityCount = new FieldVisibilityCountStatistic(getName());
fieldVisibilityCount.setBinningStrategy(new DataTypeBinningStrategy());
fieldVisibilityCount.setInternal();
statistics.add(fieldVisibilityCount);
RowRangeHistogramStatistic rowRangeHistogram = new RowRangeHistogramStatistic(getName());
rowRangeHistogram.setBinningStrategy(new CompositeBinningStrategy(new DataTypeBinningStrategy(), new PartitionBinningStrategy()));
rowRangeHistogram.setInternal();
statistics.add(rowRangeHistogram);
return statistics;
}
use of org.locationtech.geowave.core.store.api.StatisticValue in project geowave by locationtech.
the class AbstractGeoWaveBasicVectorIT method testStats.
@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
// In the multithreaded case, only test min/max and count. Stats will be
// ingested/ in a different order and will not match.
final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
final Map<String, StatisticsCache> statsCache = new HashMap<>();
final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
for (final URL inputFile : inputFiles) {
LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
while (dataIterator.hasNext()) {
final GeoWaveData<SimpleFeature> data = dataIterator.next();
final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
// it should be a statistical data adapter
if (adapter instanceof DefaultStatisticsProvider) {
StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
if (cachedValues == null) {
cachedValues = new StatisticsCache(adapter, crs);
statsCache.put(adapter.getTypeName(), cachedValues);
}
cachedValues.entryIngested(data.getValue());
}
}
}
}
final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
Assert.assertNotNull(cachedValue);
final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
int statsCount = 0;
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
while (statsIterator.hasNext()) {
statsIterator.next();
statsCount++;
}
}
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
while (statsIterator.hasNext()) {
statsIterator.next();
statsCount++;
}
}
Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
StatisticValue<Object> actual;
if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
} else {
actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
}
assertEquals(expectedValues.getValue().getValue(), actual.getValue());
}
}
// finally check the one stat that is more manually calculated -
// the bounding box
StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
validateBBox(bboxStat.getValue(), cachedValue);
// now make sure it works without giving field name because there is only one geometry field
// anyways
query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
validateBBox(bboxStat.getValue(), cachedValue);
final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
}
}
use of org.locationtech.geowave.core.store.api.StatisticValue in project geowave by locationtech.
the class DataStatisticsStoreImpl method removeStatisticValues.
@SuppressWarnings("unchecked")
@Override
public boolean removeStatisticValues(final Statistic<? extends StatisticValue<?>> statistic) {
if (statistic.getBinningStrategy() == null) {
return removeStatisticValue(statistic);
}
// TODO: The performance of this operation could be improved if primary ID prefix queries were
// allowed during delete.
boolean deleted = false;
final List<ByteArray> binsToRemove = Lists.newLinkedList();
try (CloseableIterator<StatisticValue<Object>> valueIter = getStatisticValues((Statistic<StatisticValue<Object>>) statistic)) {
while (valueIter.hasNext()) {
final ByteArray bin = valueIter.next().getBin();
binsToRemove.add(bin);
}
}
for (final ByteArray bin : binsToRemove) {
deleted = deleted || removeStatisticValue(statistic, bin);
}
return deleted;
}
use of org.locationtech.geowave.core.store.api.StatisticValue in project geowave by locationtech.
the class ListStatTypesCommand method computeResults.
@Override
public Void computeResults(final OperationParams params) {
if (parameters.isEmpty()) {
listAllRegisteredStatistics(params.getConsole());
return null;
}
final String storeName = parameters.get(0);
// Attempt to load store.
final DataStorePluginOptions storeOptions = CLIUtils.loadStore(storeName, getGeoWaveConfigFile(params), params.getConsole());
final DataStore dataStore = storeOptions.createDataStore();
if ((indexName != null) && (typeName != null)) {
throw new ParameterException("Specify either index name or type name, not both.");
}
final Index index = indexName != null ? dataStore.getIndex(indexName) : null;
if ((indexName != null) && (index == null)) {
throw new ParameterException("Unable to find index: " + indexName);
}
final DataTypeAdapter<?> adapter = typeName != null ? dataStore.getType(typeName) : null;
if ((typeName != null) && (adapter == null)) {
throw new ParameterException("Unrecognized type name: " + typeName);
}
final Map<String, List<? extends Statistic<? extends StatisticValue<?>>>> indexStats = Maps.newHashMap();
final Map<String, List<? extends Statistic<? extends StatisticValue<?>>>> adapterStats = Maps.newHashMap();
final Map<String, Map<String, List<? extends Statistic<? extends StatisticValue<?>>>>> fieldStats = Maps.newHashMap();
boolean hasAdapters = false;
if (adapter == null) {
if (index != null) {
indexStats.put(index.getName(), StatisticsRegistry.instance().getRegisteredIndexStatistics(index.getClass()));
} else {
final DataTypeAdapter<?>[] adapters = dataStore.getTypes();
for (final DataTypeAdapter<?> dataAdapter : adapters) {
hasAdapters = true;
adapterStats.put(dataAdapter.getTypeName(), StatisticsRegistry.instance().getRegisteredDataTypeStatistics(dataAdapter.getDataClass()));
fieldStats.put(dataAdapter.getTypeName(), StatisticsRegistry.instance().getRegisteredFieldStatistics(dataAdapter, fieldName));
}
final Index[] indices = dataStore.getIndices();
for (final Index idx : indices) {
indexStats.put(idx.getName(), StatisticsRegistry.instance().getRegisteredIndexStatistics(idx.getClass()));
}
}
} else {
hasAdapters = true;
adapterStats.put(adapter.getTypeName(), StatisticsRegistry.instance().getRegisteredDataTypeStatistics(adapter.getDataClass()));
fieldStats.put(adapter.getTypeName(), StatisticsRegistry.instance().getRegisteredFieldStatistics(adapter, fieldName));
}
final ConsoleTablePrinter printer = new ConsoleTablePrinter(0, Integer.MAX_VALUE, params.getConsole());
if (hasAdapters) {
displayIndexStats(printer, indexStats);
displayAdapterStats(printer, adapterStats);
displayFieldStats(printer, fieldStats);
displayBinningStrategies(printer);
} else {
params.getConsole().println("There are no types in the data store.");
}
return null;
}
use of org.locationtech.geowave.core.store.api.StatisticValue in project geowave by locationtech.
the class ListStatsCommand method performStatsCommand.
@Override
protected boolean performStatsCommand(final DataStorePluginOptions storeOptions, final StatsCommandLineOptions statsOptions, final Console console) throws IOException {
final DataStatisticsStore statsStore = storeOptions.createDataStatisticsStore();
final IndexStore indexStore = storeOptions.createIndexStore();
final String[] authorizations = getAuthorizations(statsOptions.getAuthorizations());
DataTypeAdapter<?> adapter = null;
if (statsOptions.getTypeName() != null) {
adapter = storeOptions.createDataStore().getType(statsOptions.getTypeName());
if (adapter == null) {
throw new ParameterException("A type called " + statsOptions.getTypeName() + " was not found.");
}
}
StatisticType<StatisticValue<Object>> statisticType = null;
if (statsOptions.getStatType() != null) {
statisticType = StatisticsRegistry.instance().getStatisticType(statsOptions.getStatType());
if (statisticType == null) {
throw new ParameterException("Unrecognized statistic type: " + statsOptions.getStatType());
}
}
List<String> headers = null;
List<Statistic<?>> statsToList = Lists.newLinkedList();
ValueTransformer transformer = null;
Predicate<StatisticValue<?>> filter;
if (statsOptions.getIndexName() != null) {
if (statisticType != null && !(statisticType instanceof IndexStatisticType)) {
throw new ParameterException("Only index statistic types can be specified when listing statistics for a specific index.");
}
Index index = indexStore.getIndex(statsOptions.getIndexName());
if (index == null) {
throw new ParameterException("An index called " + statsOptions.getIndexName() + " was not found.");
}
headers = Lists.newArrayList("Statistic", "Tag", "Bin", "Value");
transformer = new ValueToRow();
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getIndexStatistics(index, statisticType, statsOptions.getTag())) {
if (adapter != null) {
stats.forEachRemaining(stat -> {
if (stat.getBinningStrategy() instanceof DataTypeBinningStrategy || (stat.getBinningStrategy() instanceof CompositeBinningStrategy && ((CompositeBinningStrategy) stat.getBinningStrategy()).usesStrategy(DataTypeBinningStrategy.class))) {
statsToList.add(stat);
}
});
filter = new IndexAdapterFilter(adapter.getTypeName());
} else {
stats.forEachRemaining(statsToList::add);
filter = null;
}
}
} else if (statsOptions.getTypeName() != null) {
filter = null;
if (statsOptions.getFieldName() != null) {
if (statisticType != null && !(statisticType instanceof FieldStatisticType)) {
throw new ParameterException("Only field statistic types can be specified when listing statistics for a specific field.");
}
headers = Lists.newArrayList("Statistic", "Tag", "Bin", "Value");
transformer = new ValueToRow();
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getFieldStatistics(adapter, statisticType, statsOptions.getFieldName(), statsOptions.getTag())) {
stats.forEachRemaining(statsToList::add);
}
} else {
if (statisticType != null && statisticType instanceof IndexStatisticType) {
throw new ParameterException("Only data type and field statistic types can be specified when listing statistics for a specific data type.");
}
headers = Lists.newArrayList("Statistic", "Tag", "Field", "Bin", "Value");
transformer = new ValueToFieldRow();
if (statisticType == null || statisticType instanceof DataTypeStatisticType) {
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getDataTypeStatistics(adapter, statisticType, statsOptions.getTag())) {
stats.forEachRemaining(statsToList::add);
}
}
if (statisticType == null || statisticType instanceof FieldStatisticType) {
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getFieldStatistics(adapter, statisticType, null, statsOptions.getTag())) {
stats.forEachRemaining(statsToList::add);
}
}
}
} else if (statsOptions.getFieldName() != null) {
throw new ParameterException("A type name must be supplied with a field name.");
} else {
filter = null;
headers = Lists.newArrayList("Index/Adapter", "Statistic", "Tag", "Field", "Bin", "Value");
transformer = new ValueToAllRow();
try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats = statsStore.getAllStatistics(statisticType)) {
stats.forEachRemaining(stat -> {
if (statsOptions.getTag() == null || stat.getTag().equals(statsOptions.getTag())) {
statsToList.add(stat);
}
});
}
}
Collections.sort(statsToList, new StatComparator());
try (StatisticsValueIterator values = new StatisticsValueIterator(statsStore, statsToList.iterator(), null, authorizations)) {
Iterator<List<Object>> rows = Iterators.transform(filter == null ? values : Iterators.filter(values, v -> filter.test(v)), transformer::transform);
if (limit != null) {
rows = Iterators.limit(rows, limit);
}
if (rows.hasNext()) {
if (csv) {
StringBuilder sb = new StringBuilder();
sb.append(Arrays.toString(headers.toArray()));
rows.forEachRemaining(row -> sb.append(Arrays.toString(row.toArray())));
retValue = sb.toString();
console.println(retValue);
} else {
console.println("Matching statistics:");
ConsoleTablePrinter printer = new ConsoleTablePrinter(0, limit != null ? limit : 30, console);
printer.print(headers, rows);
}
} else {
console.println("No matching statistics were found.");
}
}
return true;
}
Aggregations