Search in sources :

Example 1 with DataStoreOperations

use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.

the class DataStoreUtils method mergeData.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
    final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
    try {
        final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> adapter : adapters) {
            if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
                mergingAdapters.put(adapter.getAdapterId(), adapter);
            }
        }
        final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
        final short[] adapterIds = new short[1];
        for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
            adapterIds[0] = adapter.getKey();
            paramsBuilder.adapterIds(adapterIds);
            try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
                final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
                final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
                while (iterator.hasNext()) {
                    iterator.next();
                }
            } catch (final Exception e) {
                LOGGER.error("Exception occurred while merging data.", e);
                throw new RuntimeException(e);
            }
        }
    } finally {
        try {
            deleter.close();
        } catch (final Exception e) {
            LOGGER.warn("Exception occurred when closing deleter.", e);
        }
    }
    return true;
}
Also used : GeoWaveKeyImpl(org.locationtech.geowave.core.store.entities.GeoWaveKeyImpl) Arrays(java.util.Arrays) ParameterException(com.beust.jcommander.ParameterException) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) HierarchicalNumericIndexStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy) FlattenedDataSet(org.locationtech.geowave.core.store.flatten.FlattenedDataSet) LoggerFactory(org.slf4j.LoggerFactory) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) PartitionBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.PartitionBinningStrategy) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) ByteBuffer(java.nio.ByteBuffer) FieldReader(org.locationtech.geowave.core.store.data.field.FieldReader) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) Lists(com.clearspring.analytics.util.Lists) NumericDimensionField(org.locationtech.geowave.core.store.dimension.NumericDimensionField) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) InternalCustomConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints.InternalCustomConstraints) Map(java.util.Map) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) GeoWaveKey(org.locationtech.geowave.core.store.entities.GeoWaveKey) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FlattenedUnreadDataSingleRow(org.locationtech.geowave.core.store.flatten.FlattenedUnreadDataSingleRow) FlattenedFieldInfo(org.locationtech.geowave.core.store.flatten.FlattenedFieldInfo) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Set(java.util.Set) UUID(java.util.UUID) DataTypeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy) List(java.util.List) VisibilityHandler(org.locationtech.geowave.core.store.api.VisibilityHandler) InsertionIds(org.locationtech.geowave.core.index.InsertionIds) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) BitmaskUtils(org.locationtech.geowave.core.store.flatten.BitmaskUtils) Entry(java.util.Map.Entry) MultiDimensionalNumericData(org.locationtech.geowave.core.index.numeric.MultiDimensionalNumericData) RowTransform(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter.RowTransform) UnconstrainedVisibilityHandler(org.locationtech.geowave.core.store.data.visibility.UnconstrainedVisibilityHandler) GeoWaveRowImpl(org.locationtech.geowave.core.store.entities.GeoWaveRowImpl) NumericIndexStrategy(org.locationtech.geowave.core.index.NumericIndexStrategy) RowRangeHistogramValue(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic.RowRangeHistogramValue) ByteArray(org.locationtech.geowave.core.index.ByteArray) IndexMetaData(org.locationtech.geowave.core.index.IndexMetaData) RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) StringUtils(org.locationtech.geowave.core.index.StringUtils) GeoWaveValueImpl(org.locationtech.geowave.core.store.entities.GeoWaveValueImpl) AdapterPersistenceEncoding(org.locationtech.geowave.core.store.adapter.AdapterPersistenceEncoding) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) SubStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy.SubStrategy) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) ArrayList(java.util.ArrayList) Mergeable(org.locationtech.geowave.core.index.Mergeable) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) VarintUtils(org.locationtech.geowave.core.index.VarintUtils) LinkedList(java.util.LinkedList) Index(org.locationtech.geowave.core.store.api.Index) MetadataDeleter(org.locationtech.geowave.core.store.operations.MetadataDeleter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) SinglePartitionInsertionIds(org.locationtech.geowave.core.index.SinglePartitionInsertionIds) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) Logger(org.slf4j.Logger) PersistentDataset(org.locationtech.geowave.core.store.data.PersistentDataset) CommonIndexModel(org.locationtech.geowave.core.store.index.CommonIndexModel) DataStore(org.locationtech.geowave.core.store.api.DataStore) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) GeoWaveValue(org.locationtech.geowave.core.store.entities.GeoWaveValue) Maps(com.google.common.collect.Maps) File(java.io.File) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) TreeMap(java.util.TreeMap) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) HintKey(org.locationtech.geowave.core.store.query.options.CommonQueryOptions.HintKey) ByteArrayUtils(org.locationtech.geowave.core.index.ByteArrayUtils) CustomIndexStrategy(org.locationtech.geowave.core.index.CustomIndexStrategy) FlattenedUnreadData(org.locationtech.geowave.core.store.flatten.FlattenedUnreadData) Collections(java.util.Collections) CompositeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) ParameterException(com.beust.jcommander.ParameterException) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter)

Example 2 with DataStoreOperations

use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.

the class DynamoDBDataStoreFactory method createStore.

@Override
public DataStore createStore(final StoreFactoryOptions options) {
    if (!(options instanceof DynamoDBOptions)) {
        throw new AssertionError("Expected " + DynamoDBOptions.class.getSimpleName());
    }
    final DynamoDBOptions opts = (DynamoDBOptions) options;
    final DataStoreOperations dynamodbOperations = helper.createOperations(opts);
    return new DynamoDBDataStore((DynamoDBOperations) dynamodbOperations);
}
Also used : DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) DynamoDBOptions(org.locationtech.geowave.datastore.dynamodb.config.DynamoDBOptions)

Example 3 with DataStoreOperations

use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.

the class GeoWaveStabilityIT method copyBadData.

@SuppressWarnings({ "unchecked", "rawtypes" })
private void copyBadData(final boolean badMetadata) throws Exception {
    final DataStoreOperations badStoreOperations = badDataStore.createDataStoreOperations();
    final DataStoreOperations storeOperations = dataStore.createDataStoreOperations();
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore indexMappingStore = dataStore.createAdapterIndexMappingStore();
    final IndexStore indexStore = dataStore.createIndexStore();
    for (final MetadataType metadataType : MetadataType.values()) {
        try (MetadataWriter writer = badStoreOperations.createMetadataWriter(metadataType)) {
            final MetadataReader reader = storeOperations.createMetadataReader(metadataType);
            try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery(null, null))) {
                while (it.hasNext()) {
                    if (badMetadata) {
                        writer.write(new BadGeoWaveMetadata(it.next()));
                    } else {
                        writer.write(it.next());
                    }
                }
            }
        } catch (final Exception e) {
            LOGGER.error("Unable to write metadata on copy", e);
        }
    }
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> adapter : adapters) {
        for (final AdapterToIndexMapping indexMapping : indexMappingStore.getIndicesForAdapter(adapter.getAdapterId())) {
            final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
            final Index index = indexMapping.getIndex(indexStore);
            final ReaderParamsBuilder bldr = new ReaderParamsBuilder(index, adapterStore, indexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
            bldr.adapterIds(new short[] { adapter.getAdapterId() });
            bldr.isClientsideRowMerging(rowMerging);
            try (RowReader<GeoWaveRow> reader = storeOperations.createReader(bldr.build())) {
                try (RowWriter writer = badStoreOperations.createWriter(index, adapter)) {
                    while (reader.hasNext()) {
                        if (!badMetadata) {
                            writer.write(new BadGeoWaveRow(reader.next()));
                        } else {
                            writer.write(reader.next());
                        }
                    }
                }
            } catch (final Exception e) {
                LOGGER.error("Unable to write metadata on copy", e);
            }
        }
    }
    try {
        badDataStore.createDataStatisticsStore().mergeStats();
    } catch (final Exception e) {
        LOGGER.info("Caught exception while merging bad stats.");
    }
}
Also used : GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) Index(org.locationtech.geowave.core.store.api.Index) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery)

Example 4 with DataStoreOperations

use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.

the class MigrationCommand method execute.

/**
 * Prep the driver & run the operation.
 */
@Override
public void execute(final OperationParams params) {
    if (parameters.size() != 1) {
        throw new ParameterException("Requires arguments: <store name>");
    }
    final String storeName = parameters.get(0);
    final StoreLoader inputStoreLoader = new StoreLoader(storeName);
    if (!inputStoreLoader.loadFromConfig(getGeoWaveConfigFile(), params.getConsole())) {
        throw new ParameterException("Cannot find store name: " + inputStoreLoader.getStoreName());
    }
    final DataStorePluginOptions storeOptions = inputStoreLoader.getDataStorePlugin();
    final DataStoreOperations operations = storeOptions.createDataStoreOperations();
    final PropertyStore propertyStore = storeOptions.createPropertyStore();
    try {
        if (!operations.metadataExists(MetadataType.ADAPTER) && !operations.metadataExists(MetadataType.INDEX)) {
            throw new ParameterException("There is no data in the data store to migrate.");
        }
    } catch (final IOException e) {
        throw new RuntimeException("Unable to determine if metadata tables exist for data store.", e);
    }
    final DataStoreProperty dataVersionProperty = propertyStore.getProperty(BaseDataStoreUtils.DATA_VERSION_PROPERTY);
    final int dataVersion = dataVersionProperty == null ? 0 : (int) dataVersionProperty.getValue();
    if (dataVersion == BaseDataStoreUtils.DATA_VERSION) {
        throw new ParameterException("The data version matches the CLI version, there are no migrations to apply.");
    }
    if (dataVersion > BaseDataStoreUtils.DATA_VERSION) {
        throw new ParameterException("The data store is using a newer serialization format.  Please update to a newer version " + "of the CLI that is compatible with the data store.");
    }
    // Do migration
    if (dataVersion < 1) {
        migrate0to1(storeOptions, operations, params.getConsole());
    }
    propertyStore.setProperty(new DataStoreProperty(BaseDataStoreUtils.DATA_VERSION_PROPERTY, BaseDataStoreUtils.DATA_VERSION));
    params.getConsole().println("Migration completed successfully!");
}
Also used : DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) DataStoreProperty(org.locationtech.geowave.core.store.DataStoreProperty) ParameterException(com.beust.jcommander.ParameterException) StoreLoader(org.locationtech.geowave.core.store.cli.store.StoreLoader) IOException(java.io.IOException) PropertyStore(org.locationtech.geowave.core.store.PropertyStore)

Example 5 with DataStoreOperations

use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.

the class BaseDataStoreUtils method verifyCLIVersion.

public static void verifyCLIVersion(final String storeName, final DataStorePluginOptions options) {
    final DataStoreOperations operations = options.createDataStoreOperations();
    final PropertyStore propertyStore = options.createPropertyStore();
    final DataStoreProperty storeVersionProperty = propertyStore.getProperty(DATA_VERSION_PROPERTY);
    if ((storeVersionProperty == null) && !hasMetadata(operations, MetadataType.ADAPTER) && !hasMetadata(operations, MetadataType.INDEX)) {
        // Nothing has been loaded into the store yet
        return;
    }
    final int storeVersion = storeVersionProperty == null ? 0 : (int) storeVersionProperty.getValue();
    if (storeVersion < DATA_VERSION) {
        throw new ParameterException("The data store '" + storeName + "' is using an older serialization format.  Either use an older " + "version of the CLI that is compatible with the data store, or migrate the data " + "store to a later version using the `geowave util migrate` command.");
    } else if (storeVersion > DATA_VERSION) {
        throw new ParameterException("The data store '" + storeName + "' is using a newer serialization format.  Please update to a " + "newer version of the CLI that is compatible with the data store.");
    }
}
Also used : DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) DataStoreProperty(org.locationtech.geowave.core.store.DataStoreProperty) ParameterException(com.beust.jcommander.ParameterException) IndexDimensionHint(org.locationtech.geowave.core.index.IndexDimensionHint) PropertyStore(org.locationtech.geowave.core.store.PropertyStore)

Aggregations

DataStoreOperations (org.locationtech.geowave.core.store.operations.DataStoreOperations)15 ParameterException (com.beust.jcommander.ParameterException)7 AdapterIndexMappingStore (org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore)4 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)4 PersistentAdapterStore (org.locationtech.geowave.core.store.adapter.PersistentAdapterStore)4 Index (org.locationtech.geowave.core.store.api.Index)4 GeoWaveMetadata (org.locationtech.geowave.core.store.entities.GeoWaveMetadata)4 MetadataQuery (org.locationtech.geowave.core.store.operations.MetadataQuery)4 DataStatisticsStore (org.locationtech.geowave.core.store.statistics.DataStatisticsStore)4 DataStore (org.locationtech.geowave.core.store.api.DataStore)3 GeoWaveRow (org.locationtech.geowave.core.store.entities.GeoWaveRow)3 IndexStore (org.locationtech.geowave.core.store.index.IndexStore)3 MetadataReader (org.locationtech.geowave.core.store.operations.MetadataReader)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 AdapterToIndexMapping (org.locationtech.geowave.core.store.AdapterToIndexMapping)2 DataStoreProperty (org.locationtech.geowave.core.store.DataStoreProperty)2 PropertyStore (org.locationtech.geowave.core.store.PropertyStore)2 InternalDataAdapter (org.locationtech.geowave.core.store.adapter.InternalDataAdapter)2 DataStorePluginOptions (org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions)2