Examples with DefaultStatisticsProvider org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider used on opensource projects

Example 1 with DefaultStatisticsProvider

use of org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider in project geowave by locationtech.

the class AbstractGeoWaveBasicVectorIT method testStats.

@SuppressWarnings("unchecked")
protected void testStats(final URL[] inputFiles, final boolean multithreaded, final CoordinateReferenceSystem crs, final Index... indices) {
    // In the multithreaded case, only test min/max and count. Stats will be
    // ingested/ in a different order and will not match.
    final LocalFileIngestPlugin<SimpleFeature> localFileIngest = new GeoToolsVectorDataStoreIngestPlugin(Filter.INCLUDE);
    final Map<String, StatisticsCache> statsCache = new HashMap<>();
    final String[] indexNames = Arrays.stream(indices).map(i -> i.getName()).toArray(i -> new String[i]);
    for (final URL inputFile : inputFiles) {
        LOGGER.warn("Calculating stats from file '" + inputFile.getPath() + "' - this may take several minutes...");
        try (final CloseableIterator<GeoWaveData<SimpleFeature>> dataIterator = localFileIngest.toGeoWaveData(inputFile, indexNames)) {
            final TransientAdapterStore adapterCache = new MemoryAdapterStore(localFileIngest.getDataAdapters());
            while (dataIterator.hasNext()) {
                final GeoWaveData<SimpleFeature> data = dataIterator.next();
                final DataTypeAdapter<SimpleFeature> adapter = data.getAdapter(adapterCache);
                // it should be a statistical data adapter
                if (adapter instanceof DefaultStatisticsProvider) {
                    StatisticsCache cachedValues = statsCache.get(adapter.getTypeName());
                    if (cachedValues == null) {
                        cachedValues = new StatisticsCache(adapter, crs);
                        statsCache.put(adapter.getTypeName(), cachedValues);
                    }
                    cachedValues.entryIngested(data.getValue());
                }
            }
        }
    }
    final DataStatisticsStore statsStore = getDataStorePluginOptions().createDataStatisticsStore();
    final PersistentAdapterStore adapterStore = getDataStorePluginOptions().createAdapterStore();
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> internalDataAdapter : adapters) {
        final FeatureDataAdapter adapter = (FeatureDataAdapter) internalDataAdapter.getAdapter();
        final StatisticsCache cachedValue = statsCache.get(adapter.getTypeName());
        Assert.assertNotNull(cachedValue);
        final Set<Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>>> expectedStats = cachedValue.statsCache.entrySet();
        int statsCount = 0;
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getDataTypeStatistics(adapter, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> statsIterator = statsStore.getFieldStatistics(adapter, null, null, null)) {
            while (statsIterator.hasNext()) {
                statsIterator.next();
                statsCount++;
            }
        }
        Assert.assertEquals("The number of stats for data adapter '" + adapter.getTypeName() + "' do not match count expected", expectedStats.size(), statsCount);
        for (final Entry<Statistic<?>, Map<ByteArray, StatisticValue<?>>> expectedStat : expectedStats) {
            for (final Entry<ByteArray, StatisticValue<?>> expectedValues : expectedStat.getValue().entrySet()) {
                StatisticValue<Object> actual;
                if (expectedValues.getKey().equals(StatisticValue.NO_BIN)) {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey());
                } else {
                    actual = statsStore.getStatisticValue((Statistic<StatisticValue<Object>>) expectedStat.getKey(), expectedValues.getKey());
                }
                assertEquals(expectedValues.getValue().getValue(), actual.getValue());
            }
        }
        // finally check the one stat that is more manually calculated -
        // the bounding box
        StatisticQuery<BoundingBoxValue, Envelope> query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).fieldName(adapter.getFeatureType().getGeometryDescriptor().getLocalName()).typeName(adapter.getTypeName()).build();
        BoundingBoxValue bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        // now make sure it works without giving field name because there is only one geometry field
        // anyways
        query = StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName(adapter.getTypeName()).build();
        bboxStat = getDataStorePluginOptions().createDataStore().aggregateStatistics(query);
        validateBBox(bboxStat.getValue(), cachedValue);
        final StatisticId<BoundingBoxValue> bboxStatId = FieldStatistic.generateStatisticId(adapter.getTypeName(), BoundingBoxStatistic.STATS_TYPE, adapter.getFeatureType().getGeometryDescriptor().getLocalName(), Statistic.INTERNAL_TAG);
        Assert.assertTrue("Unable to remove individual stat", statsStore.removeStatistic(statsStore.getStatisticById(bboxStatId)));
        Assert.assertNull("Individual stat was not successfully removed", statsStore.getStatisticById(bboxStatId));
    }
}

Example 2 with DefaultStatisticsProvider

use of org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider in project geowave by locationtech.

the class RecalculateStatsCommand method performStatsCommand.

@Override
protected boolean performStatsCommand(final DataStorePluginOptions storeOptions, final StatsCommandLineOptions statsOptions, final Console console) throws IOException {
    final DataStore dataStore = storeOptions.createDataStore();
    final DataStatisticsStore statStore = storeOptions.createDataStatisticsStore();
    final IndexStore indexStore = storeOptions.createIndexStore();
    if (all) {
        // check for legacy stats table and if it exists, delete it and add all default stats
        final DataStoreOperations ops = storeOptions.createDataStoreOperations();
        final MetadataReader reader = ops.createMetadataReader(MetadataType.LEGACY_STATISTICS);
        boolean legacyStatsExists;
        // implementation to check for at least one row
        try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery(null, null))) {
            legacyStatsExists = it.hasNext();
        }
        if (legacyStatsExists) {
            console.println("Found legacy stats prior to v1.3. Deleting and recalculating all default stats as a migration to v" + VersionUtils.getVersion() + ".");
            // first let's do the add just to make sure things are in working order prior to deleting
            // legacy stats
            console.println("Adding default statistics...");
            final List<Statistic<?>> defaultStatistics = new ArrayList<>();
            for (final Index index : dataStore.getIndices()) {
                if (index instanceof DefaultStatisticsProvider) {
                    defaultStatistics.addAll(((DefaultStatisticsProvider) index).getDefaultStatistics());
                }
            }
            for (final DataTypeAdapter<?> adapter : dataStore.getTypes()) {
                final DefaultStatisticsProvider defaultStatProvider = BaseDataStoreUtils.getDefaultStatisticsProvider(adapter);
                if (defaultStatProvider != null) {
                    defaultStatistics.addAll(defaultStatProvider.getDefaultStatistics());
                }
            }
            dataStore.addEmptyStatistic(defaultStatistics.toArray(new Statistic[defaultStatistics.size()]));
            console.println("Deleting legacy statistics...");
            try (MetadataDeleter deleter = ops.createMetadataDeleter(MetadataType.LEGACY_STATISTICS)) {
                deleter.delete(new MetadataQuery(null, null));
            } catch (final Exception e) {
                LOGGER.warn("Error deleting legacy statistics", e);
            }
            // Clear out all options so that all stats get recalculated.
            statsOptions.setIndexName(null);
            statsOptions.setTypeName(null);
            statsOptions.setFieldName(null);
            statsOptions.setStatType(null);
            statsOptions.setTag(null);
        }
    }
    final List<Statistic<? extends StatisticValue<?>>> toRecalculate = statsOptions.resolveMatchingStatistics(dataStore, statStore, indexStore);
    if (toRecalculate.isEmpty()) {
        throw new ParameterException("A matching statistic could not be found");
    } else if ((toRecalculate.size() > 1) && !all) {
        throw new ParameterException("Multiple statistics matched the given parameters.  If this is intentional, " + "supply the --all option, otherwise provide additional parameters to " + "specify which statistic to recalculate.");
    }
    final Statistic<?>[] toRecalcArray = toRecalculate.toArray(new Statistic<?>[toRecalculate.size()]);
    dataStore.recalcStatistic(toRecalcArray);
    console.println(toRecalculate.size() + " statistic" + (toRecalculate.size() == 1 ? " was" : "s were") + " successfully recalculated.");
    return true;
}

Example 3 with DefaultStatisticsProvider

use of org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider in project geowave by locationtech.

the class MigrationCommand method migrate0to1.

public void migrate0to1(final DataStorePluginOptions options, final DataStoreOperations operations, final Console console) {
    console.println("Migration 1.x -> 2.x");
    final DataStore dataStore = options.createDataStore();
    console.println("  Migrating data type adapters...");
    final PersistentAdapterStore adapterStore = options.createAdapterStore();
    final List<Short> adapterIDs = Lists.newArrayList();
    int migratedAdapters = 0;
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> adapter : adapters) {
        adapterIDs.add(adapter.getAdapterId());
        if (adapter instanceof LegacyInternalDataAdapterWrapper) {
            adapterStore.removeAdapter(adapter.getAdapterId());
            // Write updated adapter
            adapterStore.addAdapter(((LegacyInternalDataAdapterWrapper<?>) adapter).getUpdatedAdapter());
            migratedAdapters++;
        } else if (adapter.getAdapter() instanceof LegacyFeatureDataAdapter) {
            final FeatureDataAdapter updatedAdapter = ((LegacyFeatureDataAdapter) adapter.getAdapter()).getUpdatedAdapter();
            final VisibilityHandler visibilityHandler = ((LegacyFeatureDataAdapter) adapter.getAdapter()).getVisibilityHandler();
            // Write updated adapter
            adapterStore.removeAdapter(adapter.getAdapterId());
            adapterStore.addAdapter(updatedAdapter.asInternalAdapter(adapter.getAdapterId(), visibilityHandler));
            migratedAdapters++;
        }
    }
    if (migratedAdapters > 0) {
        console.println("    Migrated " + migratedAdapters + " data type adapters.");
    } else {
        console.println("    No data type adapters needed to be migrated.");
    }
    console.println("  Migrating indices...");
    final IndexStore indexStore = options.createIndexStore();
    int migratedIndices = 0;
    try (CloseableIterator<Index> indices = indexStore.getIndices()) {
        while (indices.hasNext()) {
            final Index index = indices.next();
            final CommonIndexModel indexModel = index.getIndexModel();
            // if the index model uses any spatial fields, update and re-write
            if ((indexModel != null) && (indexModel instanceof BasicIndexModel)) {
                final NumericDimensionField<?>[] oldFields = indexModel.getDimensions();
                final NumericDimensionField<?>[] updatedFields = new NumericDimensionField<?>[oldFields.length];
                boolean updated = false;
                for (int i = 0; i < oldFields.length; i++) {
                    if (oldFields[i] instanceof LegacySpatialField) {
                        updatedFields[i] = ((LegacySpatialField<?>) oldFields[i]).getUpdatedField(index);
                        updated = true;
                    } else {
                        updatedFields[i] = oldFields[i];
                    }
                }
                if (updated) {
                    ((BasicIndexModel) indexModel).init(updatedFields);
                    indexStore.removeIndex(index.getName());
                    indexStore.addIndex(index);
                    migratedIndices++;
                }
            }
        }
    }
    if (migratedIndices > 0) {
        console.println("    Migrated " + migratedIndices + " indices.");
    } else {
        console.println("    No indices needed to be migrated.");
    }
    console.println("  Migrating index mappings...");
    // Rewrite adapter to index mappings
    final LegacyAdapterIndexMappingStore legacyIndexMappings = new LegacyAdapterIndexMappingStore(operations, options.getFactoryOptions().getStoreOptions());
    final AdapterIndexMappingStore indexMappings = options.createAdapterIndexMappingStore();
    console.println("    Writing new mappings...");
    int indexMappingCount = 0;
    for (final Short adapterId : adapterIDs) {
        final LegacyAdapterToIndexMapping mapping = legacyIndexMappings.getIndicesForAdapter(adapterId);
        final InternalDataAdapter<?> adapter = adapterStore.getAdapter(adapterId);
        for (final String indexName : mapping.getIndexNames()) {
            indexMappings.addAdapterIndexMapping(BaseDataStoreUtils.mapAdapterToIndex(adapter, indexStore.getIndex(indexName)));
            indexMappingCount++;
        }
    }
    if (indexMappingCount > 0) {
        console.println("    Migrated " + indexMappingCount + " index mappings.");
        console.println("    Deleting legacy index mappings...");
        try (MetadataDeleter deleter = operations.createMetadataDeleter(MetadataType.LEGACY_INDEX_MAPPINGS)) {
            deleter.delete(new MetadataQuery(null));
        } catch (final Exception e) {
            LOGGER.warn("Error deleting legacy index mappings", e);
        }
    } else {
        console.println("    No index mappings to migrate.");
    }
    // Update statistics
    console.println("  Migrating statistics...");
    final List<Statistic<?>> defaultStatistics = new ArrayList<>();
    for (final Index index : dataStore.getIndices()) {
        if (index instanceof DefaultStatisticsProvider) {
            defaultStatistics.addAll(((DefaultStatisticsProvider) index).getDefaultStatistics());
        }
    }
    for (final DataTypeAdapter<?> adapter : dataStore.getTypes()) {
        final DefaultStatisticsProvider defaultStatProvider = BaseDataStoreUtils.getDefaultStatisticsProvider(adapter);
        if (defaultStatProvider != null) {
            defaultStatistics.addAll(defaultStatProvider.getDefaultStatistics());
        }
    }
    console.println("    Calculating updated statistics...");
    dataStore.addStatistic(defaultStatistics.toArray(new Statistic[defaultStatistics.size()]));
    console.println("    Deleting legacy statistics...");
    try (MetadataDeleter deleter = operations.createMetadataDeleter(MetadataType.LEGACY_STATISTICS)) {
        deleter.delete(new MetadataQuery(null));
    } catch (final Exception e) {
        LOGGER.warn("Error deleting legacy statistics", e);
    }
}