Search in sources :

Example 1 with MetadataWriter

use of org.locationtech.geowave.core.store.operations.MetadataWriter in project geowave by locationtech.

the class GeoWaveStabilityIT method copyBadData.

@SuppressWarnings({ "unchecked", "rawtypes" })
private void copyBadData(final boolean badMetadata) throws Exception {
    final DataStoreOperations badStoreOperations = badDataStore.createDataStoreOperations();
    final DataStoreOperations storeOperations = dataStore.createDataStoreOperations();
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore indexMappingStore = dataStore.createAdapterIndexMappingStore();
    final IndexStore indexStore = dataStore.createIndexStore();
    for (final MetadataType metadataType : MetadataType.values()) {
        try (MetadataWriter writer = badStoreOperations.createMetadataWriter(metadataType)) {
            final MetadataReader reader = storeOperations.createMetadataReader(metadataType);
            try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery(null, null))) {
                while (it.hasNext()) {
                    if (badMetadata) {
                        writer.write(new BadGeoWaveMetadata(it.next()));
                    } else {
                        writer.write(it.next());
                    }
                }
            }
        } catch (final Exception e) {
            LOGGER.error("Unable to write metadata on copy", e);
        }
    }
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> adapter : adapters) {
        for (final AdapterToIndexMapping indexMapping : indexMappingStore.getIndicesForAdapter(adapter.getAdapterId())) {
            final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
            final Index index = indexMapping.getIndex(indexStore);
            final ReaderParamsBuilder bldr = new ReaderParamsBuilder(index, adapterStore, indexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
            bldr.adapterIds(new short[] { adapter.getAdapterId() });
            bldr.isClientsideRowMerging(rowMerging);
            try (RowReader<GeoWaveRow> reader = storeOperations.createReader(bldr.build())) {
                try (RowWriter writer = badStoreOperations.createWriter(index, adapter)) {
                    while (reader.hasNext()) {
                        if (!badMetadata) {
                            writer.write(new BadGeoWaveRow(reader.next()));
                        } else {
                            writer.write(reader.next());
                        }
                    }
                }
            } catch (final Exception e) {
                LOGGER.error("Unable to write metadata on copy", e);
            }
        }
    }
    try {
        badDataStore.createDataStatisticsStore().mergeStats();
    } catch (final Exception e) {
        LOGGER.info("Caught exception while merging bad stats.");
    }
}
Also used : GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) Index(org.locationtech.geowave.core.store.api.Index) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery)

Example 2 with MetadataWriter

use of org.locationtech.geowave.core.store.operations.MetadataWriter in project geowave by locationtech.

the class AbstractGeoWavePersistence method addObject.

protected void addObject(final ByteArray id, final ByteArray secondaryId, final T object) {
    addObjectToCache(id, secondaryId, object);
    try (final MetadataWriter writer = operations.createMetadataWriter(getType())) {
        if (writer != null) {
            final GeoWaveMetadata metadata = new GeoWaveMetadata(id.getBytes(), secondaryId != null ? secondaryId.getBytes() : null, getVisibility(object), getValue(object));
            writer.write(metadata);
        }
    } catch (final Exception e) {
        LOGGER.warn("Unable to close metadata writer", e);
        e.printStackTrace();
    }
}
Also used : GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) IOException(java.io.IOException)

Example 3 with MetadataWriter

use of org.locationtech.geowave.core.store.operations.MetadataWriter in project geowave by locationtech.

the class BaseDataStore method copyTo.

@Override
public void copyTo(final DataStore other) {
    if (other instanceof BaseDataStore) {
        // efficiently copy underlying GeoWaveRow and GeoWaveMetadata
        for (final MetadataType metadataType : MetadataType.values()) {
            try (MetadataWriter writer = ((BaseDataStore) other).baseOperations.createMetadataWriter(metadataType)) {
                final MetadataReader reader = baseOperations.createMetadataReader(metadataType);
                try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery())) {
                    while (it.hasNext()) {
                        writer.write(it.next());
                    }
                }
            } catch (final Exception e) {
                LOGGER.error("Unable to write metadata on copy", e);
            }
        }
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> adapter : adapters) {
            final AdapterToIndexMapping[] mappings = indexMappingStore.getIndicesForAdapter(adapter.getAdapterId());
            for (final AdapterToIndexMapping mapping : mappings) {
                final Index index = mapping.getIndex(indexStore);
                final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
                final ReaderParamsBuilder<GeoWaveRow> bldr = new ReaderParamsBuilder<>(index, adapterStore, indexMappingStore, internalAdapterStore, rowMerging ? new GeoWaveRowMergingTransform(BaseDataStoreUtils.getRowMergingAdapter(adapter), adapter.getAdapterId()) : GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
                bldr.adapterIds(new short[] { adapter.getAdapterId() });
                bldr.isClientsideRowMerging(rowMerging);
                try (RowReader<GeoWaveRow> reader = baseOperations.createReader(bldr.build())) {
                    try (RowWriter writer = ((BaseDataStore) other).baseOperations.createWriter(index, adapter)) {
                        while (reader.hasNext()) {
                            writer.write(reader.next());
                        }
                    }
                } catch (final Exception e) {
                    LOGGER.error("Unable to write metadata on copy", e);
                }
            }
        }
    } else {
        final DataTypeAdapter<?>[] sourceTypes = getTypes();
        // add all the types that the destination store doesn't have yet
        final DataTypeAdapter<?>[] destTypes = other.getTypes();
        for (int i = 0; i < sourceTypes.length; i++) {
            boolean found = false;
            for (int k = 0; k < destTypes.length; k++) {
                if (destTypes[k].getTypeName().compareTo(sourceTypes[i].getTypeName()) == 0) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                other.addType(sourceTypes[i]);
            }
        }
        // add the indices for each type
        for (int i = 0; i < sourceTypes.length; i++) {
            final String typeName = sourceTypes[i].getTypeName();
            final short adapterId = internalAdapterStore.getAdapterId(typeName);
            final AdapterToIndexMapping[] indexMappings = indexMappingStore.getIndicesForAdapter(adapterId);
            final Index[] indices = Arrays.stream(indexMappings).map(mapping -> mapping.getIndex(indexStore)).toArray(Index[]::new);
            other.addIndex(typeName, indices);
            final QueryBuilder<?, ?> qb = QueryBuilder.newBuilder().addTypeName(typeName);
            try (CloseableIterator<?> it = query(qb.build())) {
                try (final Writer<Object> writer = other.createWriter(typeName)) {
                    while (it.hasNext()) {
                        writer.write(it.next());
                    }
                }
            }
        }
    }
}
Also used : FieldVisibilityCountValue(org.locationtech.geowave.core.store.statistics.index.FieldVisibilityCountStatistic.FieldVisibilityCountValue) Arrays(java.util.Arrays) MemoryPersistentAdapterStore(org.locationtech.geowave.core.store.memory.MemoryPersistentAdapterStore) Maps(com.beust.jcommander.internal.Maps) IndexDependentDataAdapter(org.locationtech.geowave.core.store.adapter.IndexDependentDataAdapter) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) PrefixIdQuery(org.locationtech.geowave.core.store.query.constraints.PrefixIdQuery) Statement(org.locationtech.geowave.core.store.query.gwql.statement.Statement) StatisticId(org.locationtech.geowave.core.store.statistics.StatisticId) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) DataIndexReaderParamsBuilder(org.locationtech.geowave.core.store.operations.DataIndexReaderParamsBuilder) Pair(org.apache.commons.lang3.tuple.Pair) Map(java.util.Map) Persistable(org.locationtech.geowave.core.index.persist.Persistable) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) DataTypeStatisticType(org.locationtech.geowave.core.store.statistics.adapter.DataTypeStatisticType) Query(org.locationtech.geowave.core.store.api.Query) StatisticQuery(org.locationtech.geowave.core.store.api.StatisticQuery) DataTypeStatistic(org.locationtech.geowave.core.store.api.DataTypeStatistic) Set(java.util.Set) ByteArrayConstraints(org.locationtech.geowave.core.store.api.BinConstraints.ByteArrayConstraints) IngestCallbackList(org.locationtech.geowave.core.store.callback.IngestCallbackList) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) StatisticUpdateCallback(org.locationtech.geowave.core.store.statistics.StatisticUpdateCallback) InternalStatisticsHelper(org.locationtech.geowave.core.store.statistics.InternalStatisticsHelper) IndexStatisticQuery(org.locationtech.geowave.core.store.statistics.query.IndexStatisticQuery) DeleteOtherIndicesCallback(org.locationtech.geowave.core.store.callback.DeleteOtherIndicesCallback) DefaultStatisticsProvider(org.locationtech.geowave.core.store.statistics.DefaultStatisticsProvider) ByteArray(org.locationtech.geowave.core.index.ByteArray) DataStoreProperty(org.locationtech.geowave.core.store.DataStoreProperty) StatisticType(org.locationtech.geowave.core.store.statistics.StatisticType) AggregationQuery(org.locationtech.geowave.core.store.api.AggregationQuery) DeleteCallbackList(org.locationtech.geowave.core.store.callback.DeleteCallbackList) BinConstraints(org.locationtech.geowave.core.store.api.BinConstraints) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AdapterAndIndexBasedAggregation(org.locationtech.geowave.core.store.query.aggregate.AdapterAndIndexBasedAggregation) IndexStatisticType(org.locationtech.geowave.core.store.statistics.index.IndexStatisticType) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) ArrayList(java.util.ArrayList) Lists(com.google.common.collect.Lists) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) QueryBuilder(org.locationtech.geowave.core.store.api.QueryBuilder) PropertyStore(org.locationtech.geowave.core.store.PropertyStore) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) FieldStatisticType(org.locationtech.geowave.core.store.statistics.field.FieldStatisticType) DataStore(org.locationtech.geowave.core.store.api.DataStore) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) IOException(java.io.IOException) MemoryAdapterIndexMappingStore(org.locationtech.geowave.core.store.memory.MemoryAdapterIndexMappingStore) DataStoreOptions(org.locationtech.geowave.core.store.DataStoreOptions) Writer(org.locationtech.geowave.core.store.api.Writer) IndexCompositeWriter(org.locationtech.geowave.core.store.index.writer.IndexCompositeWriter) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) DedupeFilter(org.locationtech.geowave.core.store.query.filter.DedupeFilter) DifferingVisibilityCountValue(org.locationtech.geowave.core.store.statistics.index.DifferingVisibilityCountStatistic.DifferingVisibilityCountValue) LoggerFactory(org.slf4j.LoggerFactory) Aggregation(org.locationtech.geowave.core.store.api.Aggregation) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) StatisticValue(org.locationtech.geowave.core.store.api.StatisticValue) ResultSet(org.locationtech.geowave.core.store.query.gwql.ResultSet) GeoWaveRowMergingTransform(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingTransform) DataIdRangeQuery(org.locationtech.geowave.core.store.query.constraints.DataIdRangeQuery) Statistic(org.locationtech.geowave.core.store.api.Statistic) FieldStatisticQuery(org.locationtech.geowave.core.store.statistics.query.FieldStatisticQuery) FieldDescriptor(org.locationtech.geowave.core.store.adapter.FieldDescriptor) BaseDataStoreIngestDriver(org.locationtech.geowave.core.store.ingest.BaseDataStoreIngestDriver) GWQLParser(org.locationtech.geowave.core.store.query.gwql.parse.GWQLParser) FieldStatistic(org.locationtech.geowave.core.store.api.FieldStatistic) RowReader(org.locationtech.geowave.core.store.operations.RowReader) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) IndexStatistic(org.locationtech.geowave.core.store.api.IndexStatistic) Collectors(java.util.stream.Collectors) ScanCallback(org.locationtech.geowave.core.store.callback.ScanCallback) DataTypeStatisticQuery(org.locationtech.geowave.core.store.statistics.query.DataTypeStatisticQuery) IndependentAdapterIndexWriter(org.locationtech.geowave.core.store.index.writer.IndependentAdapterIndexWriter) Sets(com.google.common.collect.Sets) List(java.util.List) VisibilityHandler(org.locationtech.geowave.core.store.api.VisibilityHandler) Entry(java.util.Map.Entry) DataIdQuery(org.locationtech.geowave.core.store.query.constraints.DataIdQuery) DataIndexUtils(org.locationtech.geowave.core.store.base.dataidx.DataIndexUtils) HashMap(java.util.HashMap) ArrayUtils(org.apache.commons.lang3.ArrayUtils) InsertionIdQuery(org.locationtech.geowave.core.store.query.constraints.InsertionIdQuery) NativeEntryIteratorWrapper(org.locationtech.geowave.core.store.util.NativeEntryIteratorWrapper) Iterators(com.google.common.collect.Iterators) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) Index(org.locationtech.geowave.core.store.api.Index) Logger(org.slf4j.Logger) IngestOptions(org.locationtech.geowave.core.store.api.IngestOptions) IngestCallback(org.locationtech.geowave.core.store.callback.IngestCallback) QueryConstraints(org.locationtech.geowave.core.store.query.constraints.QueryConstraints) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) DuplicateDeletionCallback(org.locationtech.geowave.core.store.callback.DuplicateDeletionCallback) Closeable(java.io.Closeable) TypeConstraintQuery(org.locationtech.geowave.core.store.query.constraints.TypeConstraintQuery) AdapterAndIndexBasedQueryConstraints(org.locationtech.geowave.core.store.query.constraints.AdapterAndIndexBasedQueryConstraints) EverythingQuery(org.locationtech.geowave.core.store.query.constraints.EverythingQuery) Collections(java.util.Collections) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) Index(org.locationtech.geowave.core.store.api.Index) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) DataIndexReaderParamsBuilder(org.locationtech.geowave.core.store.operations.DataIndexReaderParamsBuilder) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) IOException(java.io.IOException) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) GeoWaveRowMergingTransform(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingTransform)

Example 4 with MetadataWriter

use of org.locationtech.geowave.core.store.operations.MetadataWriter in project geowave by locationtech.

the class InternalAdapterStoreImpl method addTypeName.

// ** this introduces a distributed race condition if multiple JVM processes
// are excuting this method simultaneously
// care should be taken to either explicitly call this from a single client
// before running a distributed job, or use a distributed locking mechanism
// so that internal Adapter Ids are consistent without any race conditions
@Override
public short addTypeName(final String typeName) {
    synchronized (MUTEX) {
        Short adapterId = internalGetAdapterId(typeName, false);
        if (adapterId != null) {
            return adapterId;
        }
        adapterId = getInitialAdapterId(typeName);
        try (final MetadataWriter writer = operations.createMetadataWriter(MetadataType.INTERNAL_ADAPTER)) {
            if (writer != null) {
                final byte[] adapterIdBytes = ByteArrayUtils.shortToByteArray(adapterId);
                writer.write(new GeoWaveMetadata(StringUtils.stringToBinary(typeName), EXTERNAL_TO_INTERNAL_ID, null, adapterIdBytes));
                writer.write(new GeoWaveMetadata(adapterIdBytes, INTERNAL_TO_EXTERNAL_ID, null, StringUtils.stringToBinary(typeName)));
            }
        } catch (final Exception e) {
            LOGGER.warn("Unable to close metadata writer", e);
        }
        return adapterId;
    }
}
Also used : GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) IOException(java.io.IOException)

Aggregations

GeoWaveMetadata (org.locationtech.geowave.core.store.entities.GeoWaveMetadata)4 MetadataWriter (org.locationtech.geowave.core.store.operations.MetadataWriter)4 IOException (java.io.IOException)3 AdapterToIndexMapping (org.locationtech.geowave.core.store.AdapterToIndexMapping)2 AdapterIndexMappingStore (org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore)2 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)2 InternalDataAdapter (org.locationtech.geowave.core.store.adapter.InternalDataAdapter)2 PersistentAdapterStore (org.locationtech.geowave.core.store.adapter.PersistentAdapterStore)2 Maps (com.beust.jcommander.internal.Maps)1 Iterators (com.google.common.collect.Iterators)1 Lists (com.google.common.collect.Lists)1 Sets (com.google.common.collect.Sets)1 Closeable (java.io.Closeable)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1