Search in sources :

Example 1 with GeoWaveRow

use of org.locationtech.geowave.core.store.entities.GeoWaveRow in project geowave by locationtech.

the class DefaultStatisticVisibility method getVisibility.

@Override
public byte[] getVisibility(final T entry, final GeoWaveRow... kvs) {
    if (kvs.length == 1 && kvs[0].getFieldValues().length == 1) {
        return kvs[0].getFieldValues()[0].getVisibility();
    }
    int lowestOrdinal = Integer.MAX_VALUE;
    byte[] lowestOrdinalVisibility = null;
    for (final GeoWaveRow kv : kvs) {
        for (final GeoWaveValue v : kv.getFieldValues()) {
            final int pos = BitmaskUtils.getLowestFieldPosition(v.getFieldMask());
            if (pos == 0) {
                return v.getVisibility();
            }
            if (pos <= lowestOrdinal) {
                lowestOrdinal = pos;
                lowestOrdinalVisibility = v.getVisibility();
            }
        }
    }
    return lowestOrdinalVisibility;
}
Also used : GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) GeoWaveValue(org.locationtech.geowave.core.store.entities.GeoWaveValue)

Example 2 with GeoWaveRow

use of org.locationtech.geowave.core.store.entities.GeoWaveRow in project geowave by locationtech.

the class DataStoreUtils method mergeData.

@SuppressWarnings({ "rawtypes", "unchecked" })
public static boolean mergeData(final DataStoreOperations operations, final Integer maxRangeDecomposition, final Index index, final PersistentAdapterStore adapterStore, final InternalAdapterStore internalAdapterStore, final AdapterIndexMappingStore adapterIndexMappingStore) {
    final RowDeleter deleter = operations.createRowDeleter(index.getName(), adapterStore, internalAdapterStore);
    try {
        final Map<Short, InternalDataAdapter> mergingAdapters = new HashMap<>();
        final InternalDataAdapter<?>[] adapters = adapterStore.getAdapters();
        for (final InternalDataAdapter<?> adapter : adapters) {
            if ((adapter.getAdapter() instanceof RowMergingDataAdapter) && (((RowMergingDataAdapter) adapter.getAdapter()).getTransform() != null)) {
                mergingAdapters.put(adapter.getAdapterId(), adapter);
            }
        }
        final ReaderParamsBuilder<GeoWaveRow> paramsBuilder = new ReaderParamsBuilder<>(index, adapterStore, adapterIndexMappingStore, internalAdapterStore, GeoWaveRowIteratorTransformer.NO_OP_TRANSFORMER).isClientsideRowMerging(true).maxRangeDecomposition(maxRangeDecomposition);
        final short[] adapterIds = new short[1];
        for (final Entry<Short, InternalDataAdapter> adapter : mergingAdapters.entrySet()) {
            adapterIds[0] = adapter.getKey();
            paramsBuilder.adapterIds(adapterIds);
            try (final RowWriter writer = operations.createWriter(index, adapter.getValue());
                final RowReader<GeoWaveRow> reader = operations.createReader(paramsBuilder.build())) {
                final RewritingMergingEntryIterator<?> iterator = new RewritingMergingEntryIterator(adapterStore, adapterIndexMappingStore, index, reader, Maps.transformValues(mergingAdapters, v -> v.getAdapter()), writer, deleter);
                while (iterator.hasNext()) {
                    iterator.next();
                }
            } catch (final Exception e) {
                LOGGER.error("Exception occurred while merging data.", e);
                throw new RuntimeException(e);
            }
        }
    } finally {
        try {
            deleter.close();
        } catch (final Exception e) {
            LOGGER.warn("Exception occurred when closing deleter.", e);
        }
    }
    return true;
}
Also used : GeoWaveKeyImpl(org.locationtech.geowave.core.store.entities.GeoWaveKeyImpl) Arrays(java.util.Arrays) ParameterException(com.beust.jcommander.ParameterException) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) HierarchicalNumericIndexStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy) FlattenedDataSet(org.locationtech.geowave.core.store.flatten.FlattenedDataSet) LoggerFactory(org.slf4j.LoggerFactory) InternalAdapterStore(org.locationtech.geowave.core.store.adapter.InternalAdapterStore) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) PartitionBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.PartitionBinningStrategy) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) ByteBuffer(java.nio.ByteBuffer) FieldReader(org.locationtech.geowave.core.store.data.field.FieldReader) MetadataQuery(org.locationtech.geowave.core.store.operations.MetadataQuery) Lists(com.clearspring.analytics.util.Lists) NumericDimensionField(org.locationtech.geowave.core.store.dimension.NumericDimensionField) CustomIndex(org.locationtech.geowave.core.store.index.CustomIndex) AdapterIndexMappingStore(org.locationtech.geowave.core.store.adapter.AdapterIndexMappingStore) InternalCustomConstraints(org.locationtech.geowave.core.store.query.constraints.CustomQueryConstraints.InternalCustomConstraints) Map(java.util.Map) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) GeoWaveKey(org.locationtech.geowave.core.store.entities.GeoWaveKey) GeoWaveMetadata(org.locationtech.geowave.core.store.entities.GeoWaveMetadata) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter) FlattenedUnreadDataSingleRow(org.locationtech.geowave.core.store.flatten.FlattenedUnreadDataSingleRow) FlattenedFieldInfo(org.locationtech.geowave.core.store.flatten.FlattenedFieldInfo) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Set(java.util.Set) UUID(java.util.UUID) DataTypeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.DataTypeBinningStrategy) List(java.util.List) VisibilityHandler(org.locationtech.geowave.core.store.api.VisibilityHandler) InsertionIds(org.locationtech.geowave.core.index.InsertionIds) IndexStore(org.locationtech.geowave.core.store.index.IndexStore) BitmaskUtils(org.locationtech.geowave.core.store.flatten.BitmaskUtils) Entry(java.util.Map.Entry) MultiDimensionalNumericData(org.locationtech.geowave.core.index.numeric.MultiDimensionalNumericData) RowTransform(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter.RowTransform) UnconstrainedVisibilityHandler(org.locationtech.geowave.core.store.data.visibility.UnconstrainedVisibilityHandler) GeoWaveRowImpl(org.locationtech.geowave.core.store.entities.GeoWaveRowImpl) NumericIndexStrategy(org.locationtech.geowave.core.index.NumericIndexStrategy) RowRangeHistogramValue(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic.RowRangeHistogramValue) ByteArray(org.locationtech.geowave.core.index.ByteArray) IndexMetaData(org.locationtech.geowave.core.index.IndexMetaData) RowRangeHistogramStatistic(org.locationtech.geowave.core.store.statistics.index.RowRangeHistogramStatistic) StringUtils(org.locationtech.geowave.core.index.StringUtils) GeoWaveValueImpl(org.locationtech.geowave.core.store.entities.GeoWaveValueImpl) AdapterPersistenceEncoding(org.locationtech.geowave.core.store.adapter.AdapterPersistenceEncoding) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) SubStrategy(org.locationtech.geowave.core.index.HierarchicalNumericIndexStrategy.SubStrategy) QueryRanges(org.locationtech.geowave.core.index.QueryRanges) AdapterToIndexMapping(org.locationtech.geowave.core.store.AdapterToIndexMapping) ArrayList(java.util.ArrayList) Mergeable(org.locationtech.geowave.core.index.Mergeable) HashSet(java.util.HashSet) DataStatisticsStore(org.locationtech.geowave.core.store.statistics.DataStatisticsStore) MetadataType(org.locationtech.geowave.core.store.operations.MetadataType) DataTypeAdapter(org.locationtech.geowave.core.store.api.DataTypeAdapter) VarintUtils(org.locationtech.geowave.core.index.VarintUtils) LinkedList(java.util.LinkedList) Index(org.locationtech.geowave.core.store.api.Index) MetadataDeleter(org.locationtech.geowave.core.store.operations.MetadataDeleter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) SinglePartitionInsertionIds(org.locationtech.geowave.core.index.SinglePartitionInsertionIds) DataStoreOperations(org.locationtech.geowave.core.store.operations.DataStoreOperations) Logger(org.slf4j.Logger) PersistentDataset(org.locationtech.geowave.core.store.data.PersistentDataset) CommonIndexModel(org.locationtech.geowave.core.store.index.CommonIndexModel) DataStore(org.locationtech.geowave.core.store.api.DataStore) MetadataReader(org.locationtech.geowave.core.store.operations.MetadataReader) GeoWaveValue(org.locationtech.geowave.core.store.entities.GeoWaveValue) Maps(com.google.common.collect.Maps) File(java.io.File) PersistentAdapterStore(org.locationtech.geowave.core.store.adapter.PersistentAdapterStore) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) DataStorePluginOptions(org.locationtech.geowave.core.store.cli.store.DataStorePluginOptions) TreeMap(java.util.TreeMap) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) HintKey(org.locationtech.geowave.core.store.query.options.CommonQueryOptions.HintKey) ByteArrayUtils(org.locationtech.geowave.core.index.ByteArrayUtils) CustomIndexStrategy(org.locationtech.geowave.core.index.CustomIndexStrategy) FlattenedUnreadData(org.locationtech.geowave.core.store.flatten.FlattenedUnreadData) Collections(java.util.Collections) CompositeBinningStrategy(org.locationtech.geowave.core.store.statistics.binning.CompositeBinningStrategy) RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) HashMap(java.util.HashMap) RowDeleter(org.locationtech.geowave.core.store.operations.RowDeleter) RowWriter(org.locationtech.geowave.core.store.operations.RowWriter) ParameterException(com.beust.jcommander.ParameterException) ReaderParamsBuilder(org.locationtech.geowave.core.store.operations.ReaderParamsBuilder) InternalDataAdapter(org.locationtech.geowave.core.store.adapter.InternalDataAdapter)

Example 3 with GeoWaveRow

use of org.locationtech.geowave.core.store.entities.GeoWaveRow in project geowave by locationtech.

the class MergingEntryIterator method getNextEncodedResult.

@Override
protected GeoWaveRow getNextEncodedResult() {
    GeoWaveRow nextResult = scannerIt.next();
    final short internalAdapterId = nextResult.getAdapterId();
    final RowMergingDataAdapter mergingAdapter = mergingAdapters.get(internalAdapterId);
    if ((mergingAdapter != null) && (mergingAdapter.getTransform() != null)) {
        final RowTransform rowTransform = getRowTransform(internalAdapterId, mergingAdapter);
        // This iterator expects a single GeoWaveRow w/ multiple fieldValues
        nextResult = mergeSingleRowValues(nextResult, rowTransform);
    }
    return nextResult;
}
Also used : RowMergingDataAdapter(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) RowTransform(org.locationtech.geowave.core.store.adapter.RowMergingDataAdapter.RowTransform)

Example 4 with GeoWaveRow

use of org.locationtech.geowave.core.store.entities.GeoWaveRow in project geowave by locationtech.

the class NativeEntryIteratorWrapper method findNext.

protected void findNext() {
    while ((nextValue == null) && hasNextScannedResult()) {
        final GeoWaveRow row = getNextEncodedResult();
        final T decodedValue = decodeRow(row, clientFilters, index);
        if (decodedValue != null) {
            nextValue = decodedValue;
            return;
        }
    }
}
Also used : GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow)

Example 5 with GeoWaveRow

use of org.locationtech.geowave.core.store.entities.GeoWaveRow in project geowave by locationtech.

the class AccumuloKeyValuePairGenerator method constructKeyValuePairs.

public List<KeyValue> constructKeyValuePairs(final T entry) {
    final List<KeyValue> keyValuePairs = new ArrayList<>();
    final GeoWaveRow[] rows = BaseDataStoreUtils.getGeoWaveRows(entry, adapter, indexMapping, index, visibilityHandler);
    if ((rows != null) && (rows.length > 0)) {
        for (final GeoWaveRow row : rows) {
            final Mutation m = AccumuloWriter.rowToMutation(row);
            for (final ColumnUpdate cu : m.getUpdates()) {
                keyValuePairs.add(new KeyValue(new Key(m.getRow(), cu.getColumnFamily(), cu.getColumnQualifier(), cu.getColumnVisibility(), cu.getTimestamp()), cu.getValue()));
            }
        }
    }
    return keyValuePairs;
}
Also used : GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) KeyValue(org.apache.accumulo.core.data.KeyValue) ColumnUpdate(org.apache.accumulo.core.data.ColumnUpdate) ArrayList(java.util.ArrayList) Mutation(org.apache.accumulo.core.data.Mutation) Key(org.apache.accumulo.core.data.Key)

Aggregations

GeoWaveRow (org.locationtech.geowave.core.store.entities.GeoWaveRow)43 ArrayList (java.util.ArrayList)25 List (java.util.List)18 CloseableIterator (org.locationtech.geowave.core.store.CloseableIterator)18 Arrays (java.util.Arrays)16 ByteArrayRange (org.locationtech.geowave.core.index.ByteArrayRange)16 ByteArray (org.locationtech.geowave.core.index.ByteArray)15 Iterator (java.util.Iterator)14 Map (java.util.Map)13 InternalAdapterStore (org.locationtech.geowave.core.store.adapter.InternalAdapterStore)13 PersistentAdapterStore (org.locationtech.geowave.core.store.adapter.PersistentAdapterStore)13 RowReader (org.locationtech.geowave.core.store.operations.RowReader)13 IOException (java.io.IOException)12 Set (java.util.Set)12 InternalDataAdapter (org.locationtech.geowave.core.store.adapter.InternalDataAdapter)12 Index (org.locationtech.geowave.core.store.api.Index)12 Logger (org.slf4j.Logger)12 LoggerFactory (org.slf4j.LoggerFactory)12 Streams (com.google.common.collect.Streams)11 HashMap (java.util.HashMap)11