use of org.locationtech.geowave.core.store.operations.RowWriter in project geowave by locationtech.
the class DataStoreUtils method mergeData.
@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
try {
final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
mergingAdapters.put(adapter.getAdapterId(), adapter);
}
}
final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
final short[] adapterIds = new short[1];
for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
adapterIds[0] = adapter.getKey();
paramsBuilder.adapterIds(adapterIds);
try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
while (iterator.hasNext()) {
iterator.next();
}
} catch (final Exception e) {
LOGGER.error("Exception occurred while merging data.", e);
throw new RuntimeException(e);
}
}
} finally {
try {
deleter.close();
} catch (final Exception e) {
LOGGER.warn("Exception occurred when closing deleter.", e);
}
}
return true;
}
use of org.locationtech.geowave.core.store.operations.RowWriter in project geowave by locationtech.
the class GeoWaveStabilityIT method copyBadData.
@SuppressWarnings({ "unchecked", "rawtypes" })
private void copyBadData(final boolean badMetadata) throws Exception {
final DataStoreOperations badStoreOperations = badDataStore.createDataStoreOperations();
final DataStoreOperations storeOperations = dataStore.createDataStoreOperations();
final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
final AdapterIndexMappingStore indexMappingStore = dataStore.createAdapterIndexMappingStore();
final IndexStore indexStore = dataStore.createIndexStore();
for (final MetadataType metadataType : MetadataType.values()) {
try (MetadataWriter writer = badStoreOperations.createMetadataWriter(metadataType)) {
final MetadataReader reader = storeOperations.createMetadataReader(metadataType);
try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery(null, null))) {
while (it.hasNext()) {
if (badMetadata) {
writer.write(new BadGeoWaveMetadata(it.next()));
} else {
writer.write(it.next());
}
}
}
} catch (final Exception e) {
LOGGER.error("Unable to write metadata on copy", e);
}
}
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
for (final AdapterToIndexMapping indexMapping : indexMappingStore.getIndicesForAdapter(adapter.getAdapterId())) {
final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
final Index index = indexMapping.getIndex(indexStore);
final ReaderParamsBuilder bldr = new ReaderParamsBuilder(index, adapterStore, indexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
bldr.adapterIds(new short[] { adapter.getAdapterId() });
bldr.isClientsideRowMerging(rowMerging);
try (RowReader<GeoWaveRow> reader = storeOperations.createReader(bldr.build())) {
try (RowWriter writer = badStoreOperations.createWriter(index, adapter)) {
while (reader.hasNext()) {
if (!badMetadata) {
writer.write(new BadGeoWaveRow(reader.next()));
} else {
writer.write(reader.next());
}
}
}
} catch (final Exception e) {
LOGGER.error("Unable to write metadata on copy", e);
}
}
}
try {
badDataStore.createDataStatisticsStore().mergeStats();
} catch (final Exception e) {
LOGGER.info("Caught exception while merging bad stats.");
}
}
use of org.locationtech.geowave.core.store.operations.RowWriter in project geowave by locationtech.
the class BaseDataStore method copyTo.
@Override
public void copyTo(final DataStore other) {
if (other instanceof BaseDataStore) {
// efficiently copy underlying GeoWaveRow and GeoWaveMetadata
for (final MetadataType metadataType : MetadataType.values()) {
try (MetadataWriter writer = ((BaseDataStore) other).baseOperations.createMetadataWriter(metadataType)) {
final MetadataReader reader = baseOperations.createMetadataReader(metadataType);
try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery())) {
while (it.hasNext()) {
writer.write(it.next());
}
}
} catch (final Exception e) {
LOGGER.error("Unable to write metadata on copy", e);
}
}
final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
for (final InternalDataAdapter<?> adapter : adapters) {
final AdapterToIndexMapping[] mappings = indexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
for (final AdapterToIndexMapping mapping : mappings) {
final Index index = mapping.getIndex(indexStore);
final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
final ReaderParamsBuilder<GeoWaveRow> bldr = new ReaderParamsBuilder<>(index, adapterStore, indexMappingStore, internalAdapterStore, rowMerging ? new GeoWaveRowMergingTransform(BaseDataStoreUtils.getRowMergingAdapter(adapter), adapter.getAdapterId()) : GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
bldr.adapterIds(new short[] { adapter.getAdapterId() });
bldr.isClientsideRowMerging(rowMerging);
try (RowReader<GeoWaveRow> reader = baseOperations.createReader(bldr.build())) {
try (RowWriter writer = ((BaseDataStore) other).baseOperations.createWriter(index, adapter)) {
while (reader.hasNext()) {
writer.write(reader.next());
}
}
} catch (final Exception e) {
LOGGER.error("Unable to write metadata on copy", e);
}
}
}
} else {
final DataTypeAdapter<?>[] sourceTypes = getTypes();
// add all the types that the destination store doesn't have yet
final DataTypeAdapter<?>[] destTypes = other.getTypes();
for (int i = 0; i < sourceTypes.length; i++) {
boolean found = false;
for (int k = 0; k < destTypes.length; k++) {
if (destTypes[k].getTypeName().compareTo(sourceTypes[i].getTypeName()) == 0) {
found = true;
break;
}
}
if (!found) {
other.addType(sourceTypes[i]);
}
}
// add the indices for each type
for (int i = 0; i < sourceTypes.length; i++) {
final String typeName = sourceTypes[i].getTypeName();
final short adapterId = internalAdapterStore.getAdapterId(typeName);
final AdapterToIndexMapping[] indexMappings = indexMappingStore.getIndicesForAdapter(adapterId);
final Index[] indices = Arrays.stream(indexMappings).map(mapping -> mapping.getIndex(indexStore)).toArray(Index[]::new);
other.addIndex(typeName, indices);
final QueryBuilder<?, ?> qb = QueryBuilder.newBuilder().addTypeName(typeName);
try (CloseableIterator<?> it = query(qb.build())) {
try (final Writer<Object> writer = other.createWriter(typeName)) {
while (it.hasNext()) {
writer.write(it.next());
}
}
}
}
}
}
Aggregations