Search in sources :

Example 1 with MetadataQuery

use of org.locationtech.geowave.core.store.operations.MetadataQuery in project geowave by locationtech.

the class DataStoreUtils method safeMetadataDelete.

public static void safeMetadataDelete(final MetadataDeleter deleter, final DataStoreOperations operations, final MetadataType metadataType, final MetadataQuery query) {
    // we need to respect visibilities although this may be much slower
    final MetadataReader reader = operations.createMetadataReader(metadataType);
    try (final CloseableIterator<GeoWaveMetadata> it = reader.query(query)) {
        while (it.hasNext()) {
            final GeoWaveMetadata entry = it.next();
            deleter.delete(new MetadataQuery(entry.getPrimaryId(), entry.getSecondaryId(), query.getAuthorizations()));
        }
    }
}
Also used : MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery)

Example 2 with MetadataQuery

use of org.locationtech.geowave.core.store.operations.MetadataQuery in project geowave by locationtech.

the class CassandraMetadataReader method query.

@Override
public CloseableIterator<GeoWaveMetadata> query(final MetadataQuery query) {
    final String tableName = operations.getMetadataTableName(metadataType);
    final String[] selectedColumns = metadataType.isStatValues() ? ArrayUtils.add(getSelectedColumns(query), CassandraMetadataWriter.VISIBILITY_KEY) : getSelectedColumns(query);
    Predicate<Row> clientFilter = null;
    if (query.isPrefix()) {
        if (query.hasPrimaryId()) {
            clientFilter = new PrimaryIDPrefixFilter(query.getPrimaryId());
        }
    }
    final Iterator<Row> rows;
    if (!query.hasPrimaryIdRanges()) {
        Select select = operations.getSelect(tableName, selectedColumns);
        if (query.hasPrimaryId() && query.isExact()) {
            select = select.whereColumn(CassandraMetadataWriter.PRIMARY_ID_KEY).isEqualTo(QueryBuilder.literal(ByteBuffer.wrap(query.getPrimaryId())));
            if (query.hasSecondaryId()) {
                select = select.whereColumn(CassandraMetadataWriter.SECONDARY_ID_KEY).isEqualTo(QueryBuilder.literal(ByteBuffer.wrap(query.getSecondaryId())));
            }
        } else if (query.hasSecondaryId()) {
            select = select.allowFiltering().whereColumn(CassandraMetadataWriter.SECONDARY_ID_KEY).isEqualTo(QueryBuilder.literal(ByteBuffer.wrap(query.getSecondaryId())));
        }
        final ResultSet rs = operations.getSession().execute(select.build());
        rows = rs.iterator();
    } else {
        rows = Iterators.concat(Arrays.stream(query.getPrimaryIdRanges()).map((r) -> {
            // TODO this is not as efficient as prepared bound statements if there are many
            // ranges, but will work for now
            Select select = operations.getSelect(tableName, selectedColumns);
            if (r.getStart() != null) {
                select = select.allowFiltering().whereColumn(CassandraMetadataWriter.PRIMARY_ID_KEY).isGreaterThanOrEqualTo(QueryBuilder.literal(ByteBuffer.wrap(r.getStart())));
            }
            if (r.getEnd() != null) {
                select = select.allowFiltering().whereColumn(CassandraMetadataWriter.PRIMARY_ID_KEY).isLessThan(QueryBuilder.literal(ByteBuffer.wrap(r.getEndAsNextPrefix())));
            }
            final ResultSet rs = operations.getSession().execute(select.build());
            return rs.iterator();
        }).iterator());
    }
    final CloseableIterator<GeoWaveMetadata> retVal = new CloseableIterator.Wrapper<>(Iterators.transform(clientFilter != null ? Iterators.filter(rows, clientFilter) : rows, result -> new GeoWaveMetadata((query.hasPrimaryId() && query.isExact()) ? query.getPrimaryId() : result.get(CassandraMetadataWriter.PRIMARY_ID_KEY, ByteBuffer.class).array(), useSecondaryId(query) ? query.getSecondaryId() : result.get(CassandraMetadataWriter.SECONDARY_ID_KEY, ByteBuffer.class).array(), getVisibility(query, result), result.get(CassandraMetadataWriter.VALUE_KEY, ByteBuffer.class).array())));
    return query.getAuthorizations() != null ? MetadataIterators.clientVisibilityFilter(retVal, query.getAuthorizations()) : retVal;
}
Also used : QueryBuilder(com.datastax.oss.driver.api.querybuilder.QueryBuilder) Arrays(java.util.Arrays) Iterator(java.util.Iterator) ResultSet(com.datastax.oss.driver.api.core.cql.ResultSet) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) ArrayUtils(org.apache.commons.lang3.ArrayUtils) ByteBuffer(java.nio.ByteBuffer) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) Iterators(com.google.common.collect.Iterators) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Predicate(com.google.common.base.Predicate) ByteArrayUtils(org.locationtech.geowave.core.index.ByteArrayUtils) MetadataIterators(org.locationtech.geowave.core.store.metadata.MetadataIterators) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) Row(com.datastax.oss.driver.api.core.cql.Row) Select(com.datastax.oss.driver.api.querybuilder.select.Select) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) ByteBuffer(java.nio.ByteBuffer) Select(com.datastax.oss.driver.api.querybuilder.select.Select) ResultSet(com.datastax.oss.driver.api.core.cql.ResultSet) Row(com.datastax.oss.driver.api.core.cql.Row)

Example 3 with MetadataQuery

use of org.locationtech.geowave.core.store.operations.MetadataQuery in project geowave by locationtech.

the class GeoWaveStabilityIT method copyBadData.

@SuppressWarnings({ "unchecked", "rawtypes" })
private void copyBadData(final boolean badMetadata) throws Exception {
    final DataStoreOperations badStoreOperations = badDataStore.createDataStoreOperations();
    final DataStoreOperations storeOperations = dataStore.createDataStoreOperations();
    final PersistentAdapterStore adapterStore = dataStore.createAdapterStore();
    final InternalAdapterStore internalAdapterStore = dataStore.createInternalAdapterStore();
    final AdapterIndexMappingStore indexMappingStore = dataStore.createAdapterIndexMappingStore();
    final IndexStore indexStore = dataStore.createIndexStore();
    for (final MetadataType metadataType : MetadataType.values()) {
        try (MetadataWriter writer = badStoreOperations.createMetadataWriter(metadataType)) {
            final MetadataReader reader = storeOperations.createMetadataReader(metadataType);
            try (CloseableIterator<GeoWaveMetadata> it = reader.query(new MetadataQuery(null, null))) {
                while (it.hasNext()) {
                    if (badMetadata) {
                        writer.write(new BadGeoWaveMetadata(it.next()));
                    } else {
                        writer.write(it.next());
                    }
                }
            }
        } catch (final Exception e) {
            LOGGER.error("Unable to write metadata on copy", e);
        }
    }
    final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
    for (final InternalDataAdapter<?> adapter : adapters) {
        for (final AdapterToIndexMapping indexMapping : indexMappingStore.getIndicesForAdapter(adapter.getAdapterId())) {
            final boolean rowMerging = BaseDataStoreUtils.isRowMerging(adapter);
            final Index index = indexMapping.getIndex(indexStore);
            final ReaderParamsBuilder bldr = new ReaderParamsBuilder(index, adapterStore, indexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER);
            bldr.adapterIds(new short[] { adapter.getAdapterId() });
            bldr.isClientsideRowMerging(rowMerging);
            try (RowReader<GeoWaveRow> reader = storeOperations.createReader(bldr.build())) {
                try (RowWriter writer = badStoreOperations.createWriter(index, adapter)) {
                    while (reader.hasNext()) {
                        if (!badMetadata) {
                            writer.write(new BadGeoWaveRow(reader.next()));
                        } else {
                            writer.write(reader.next());
                        }
                    }
                }
            } catch (final Exception e) {
                LOGGER.error("Unable to write metadata on copy", e);
            }
        }
    }
    try {
        badDataStore.createDataStatisticsStore().mergeStats();
    } catch (final Exception e) {
        LOGGER.info("Caught exception while merging bad stats.");
    }
}
Also used : GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) Index(org.locationtech.geowave.core.store.api.Index) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery)

Example 4 with MetadataQuery

use of org.locationtech.geowave.core.store.operations.MetadataQuery in project geowave by locationtech.

the class DataStatisticsStoreImpl method createStatisticValueReader.

private <V extends StatisticValue<R>, R> StatisticValueReader<V, R> createStatisticValueReader(final Statistic<V> statistic, final ByteArray bin, final boolean exact, final String... authorizations) {
    final byte[] primaryId;
    if ((bin == null) && !exact) {
        primaryId = StatisticValue.getValueId(statistic.getId(), new byte[0]);
    } else {
        primaryId = StatisticValue.getValueId(statistic.getId(), bin);
    }
    final MetadataQuery query = new MetadataQuery(primaryId, statistic.getId().getGroupId().getBytes(), !exact, authorizations);
    return new StatisticValueReader<>(operations.createMetadataReader(MetadataType.STATISTIC_VALUES).query(query), statistic);
}
Also used : StatisticValueReader(org.locationtech.geowave.core.store.statistics.StatisticValueReader) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery)

Example 5 with MetadataQuery

use of org.locationtech.geowave.core.store.operations.MetadataQuery in project geowave by locationtech.

the class InternalAdapterStoreImpl method getAdapterIds.

@Override
public short[] getAdapterIds() {
    final MetadataReader reader = getReader(false);
    if (reader == null) {
        return new short[0];
    }
    final CloseableIterator<GeoWaveMetadata> results = reader.query(new MetadataQuery(EXTERNAL_TO_INTERNAL_ID));
    try (CloseableIterator<Short> it = new CloseableIteratorWrapper<>(results, Iterators.transform(results, input -> ByteArrayUtils.byteArrayToShort(input.getValue())))) {
        return ArrayUtils.toPrimitive(Iterators.toArray(it, Short.class));
    }
}
Also used : ByteArray(org.locationtech.geowave.core.index.ByteArray) BiMap(com.google.common.collect.BiMap) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) Logger(org.slf4j.Logger) StringUtils(org.locationtech.geowave.core.index.StringUtils) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) LoggerFactory(org.slf4j.LoggerFactory) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) IOException(java.io.IOException) Maps(com.google.common.collect.Maps) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) Iterators(com.google.common.collect.Iterators) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) HashBiMap(com.google.common.collect.HashBiMap) MetadataWriter(org.locationtech.geowave.core.store.operations.MetadataWriter) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ByteArrayUtils(org.locationtech.geowave.core.index.ByteArrayUtils) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) ArrayUtils(org.apache.commons.lang.ArrayUtils) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery)

Aggregations

MetadataQuery (org.locationtech.geowave.core.store.operations.MetadataQuery)19 GeoWaveMetadata (org.locationtech.geowave.core.store.entities.GeoWaveMetadata)17 MetadataReader (org.locationtech.geowave.core.store.operations.MetadataReader)15 MetadataType (org.locationtech.geowave.core.store.operations.MetadataType)9 IOException (java.io.IOException)8 CloseableIterator (org.locationtech.geowave.core.store.CloseableIterator)8 Iterators (com.google.common.collect.Iterators)7 Arrays (java.util.Arrays)6 ByteArrayUtils (org.locationtech.geowave.core.index.ByteArrayUtils)6 CloseableIteratorWrapper (org.locationtech.geowave.core.store.CloseableIteratorWrapper)6 DataStoreOperations (org.locationtech.geowave.core.store.operations.DataStoreOperations)6 Logger (org.slf4j.Logger)6 LoggerFactory (org.slf4j.LoggerFactory)6 ArrayList (java.util.ArrayList)5 Index (org.locationtech.geowave.core.store.api.Index)5 Iterator (java.util.Iterator)4 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)4 IndexStore (org.locationtech.geowave.core.store.index.IndexStore)4 MetadataWriter (org.locationtech.geowave.core.store.operations.MetadataWriter)4 Collections (java.util.Collections)3