use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.
the class DataStoreUtils method mergeData.
@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
try {
final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
mergingAdapters.put(adapter.getAdapterId(), adapter);
}
}
final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
final short[] adapterIds = new short[1];
for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
adapterIds[0] = adapter.getKey();
paramsBuilder.adapterIds(adapterIds);
try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
while (iterator.hasNext()) {
iterator.next();
}
} catch (final Exception e) {
LOGGER.error("Exception occurred while merging data.", e);
throw new RuntimeException(e);
}
}
} finally {
try {
deleter.close();
} catch (final Exception e) {
LOGGER.warn("Exception occurred when closing deleter.", e);
}
}
return true;
}
use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.
the class DynamoDBDataStoreFactory method createStore.
@Override
public DataStore createStore(final StoreFactoryOptions options) {
if (!(options instanceof DynamoDBOptions)) {
throw new AssertionError("Expected " + DynamoDBOptions.class.getSimpleName());
}
final DynamoDBOptions opts = (DynamoDBOptions) options;
final DataStoreOperations dynamodbOperations = helper.createOperations(opts);
return new DynamoDBDataStore((DynamoDBOperations) dynamodbOperations);
}
use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.
the class GeoWaveStabilityIT method copyBadData.
@SuppressWarnings({ "unchecked", "rawtypes" })
private void copyBadData(final boolean badMetadata) throws Exception {
final DataStoreOperations badStoreOperations = badDataStore.createDataStoreOperations();
final DataStoreOperations storeOperations = dataStore.createDataStoreOperations();
final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
final AdapterIndexMappingStore indexMappingStore = dataStore.createAdapterIndexMappingStore();
final IndexStore indexStore = dataStore.createIndexStore();
for (final MetadataType metadataType : MetadataType.values()) {
try (MetadataWriter writer = badStoreOperations.createMetadataWriter(metadataType)) {
final MetadataReader reader = storeOperations.createMetadataReader(metadataType);
try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery(null, null))) {
while (it.hasNext()) {
if (badMetadata) {
writer.write(new BadGeoWaveMetadata(it.next()));
} else {
writer.write(it.next());
}
}
}
} catch (final Exception e) {
LOGGER.error("Unable to write metadata on copy", e);
}
}
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
for (final AdapterToIndexMapping indexMapping : indexMappingStore.getIndicesForAdapter(adapter.getAdapterId())) {
final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
final Index index = indexMapping.getIndex(indexStore);
final ReaderParamsBuilder bldr = new ReaderParamsBuilder(index, adapterStore, indexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
bldr.adapterIds(new short[] { adapter.getAdapterId() });
bldr.isClientsideRowMerging(rowMerging);
try (RowReader<GeoWaveRow> reader = storeOperations.createReader(bldr.build())) {
try (RowWriter writer = badStoreOperations.createWriter(index, adapter)) {
while (reader.hasNext()) {
if (!badMetadata) {
writer.write(new BadGeoWaveRow(reader.next()));
} else {
writer.write(reader.next());
}
}
}
} catch (final Exception e) {
LOGGER.error("Unable to write metadata on copy", e);
}
}
}
try {
badDataStore.createDataStatisticsStore().mergeStats();
} catch (final Exception e) {
LOGGER.info("Caught exception while merging bad stats.");
}
}
use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.
the class MigrationCommand method execute.
/**
* Prep the driver & run the operation.
*/
@Override
public void execute(final OperationParams params) {
if (parameters.size() != 1) {
throw new ParameterException("Requires arguments: <store name>");
}
final String storeName = parameters.get(0);
final StoreLoader inputStoreLoader = new StoreLoader(storeName);
if (!inputStoreLoader.loadFromConfig(getGeoWaveConfigFile(), params.getConsole())) {
throw new ParameterException("Cannot find store name: " + inputStoreLoader.getStoreName());
}
final DataStorePluginOptions storeOptions = inputStoreLoader.getDataStorePlugin();
final DataStoreOperations operations = storeOptions.createDataStoreOperations();
final PropertyStore propertyStore = storeOptions.createPropertyStore();
try {
if (!operations.metadataExists(MetadataType.ADAPTER) && !operations.metadataExists(MetadataType.INDEX)) {
throw new ParameterException("There is no data in the data store to migrate.");
}
} catch (final IOException e) {
throw new RuntimeException("Unable to determine if metadata tables exist for data store.", e);
}
final DataStoreProperty dataVersionProperty = propertyStore.getProperty(BaseDataStoreUtils.DATA_VERSION_PROPERTY);
final int dataVersion = dataVersionProperty == null ? 0 : (int) dataVersionProperty.getValue();
if (dataVersion == BaseDataStoreUtils.DATA_VERSION) {
throw new ParameterException("The data version matches the CLI version, there are no migrations to apply.");
}
if (dataVersion > BaseDataStoreUtils.DATA_VERSION) {
throw new ParameterException("The data store is using a newer serialization format. Please update to a newer version " + "of the CLI that is compatible with the data store.");
}
// Do migration
if (dataVersion < 1) {
migrate0to1(storeOptions, operations, params.getConsole());
}
propertyStore.setProperty(new DataStoreProperty(BaseDataStoreUtils.DATA_VERSION_PROPERTY, BaseDataStoreUtils.DATA_VERSION));
params.getConsole().println("Migration completed successfully!");
}
use of org.locationtech.geowave.core.store.operations.DataStoreOperations in project geowave by locationtech.
the class BaseDataStoreUtils method verifyCLIVersion.
public static void verifyCLIVersion(final String storeName, final DataStorePluginOptions options) {
final DataStoreOperations operations = options.createDataStoreOperations();
final PropertyStore propertyStore = options.createPropertyStore();
final DataStoreProperty storeVersionProperty = propertyStore.getProperty(DATA_VERSION_PROPERTY);
if ((storeVersionProperty == null) && !hasMetadata(operations, MetadataType.ADAPTER) && !hasMetadata(operations, MetadataType.INDEX)) {
// Nothing has been loaded into the store yet
return;
}
final int storeVersion = storeVersionProperty == null ? 0 : (int) storeVersionProperty.getValue();
if (storeVersion < DATA_VERSION) {
throw new ParameterException("The data store '" + storeName + "' is using an older serialization format. Either use an older " + "version of the CLI that is compatible with the data store, or migrate the data " + "store to a later version using the `geowave util migrate` command.");
} else if (storeVersion > DATA_VERSION) {
throw new ParameterException("The data store '" + storeName + "' is using a newer serialization format. Please update to a " + "newer version of the CLI that is compatible with the data store.");
}
}
Aggregations