Search in sources :

Example 1 with RocksDBClient

use of org.locationtech.geowave.datastore.rocksdb.util.RocksDBClient in project geowave by locationtech.

the class RocksDBReader method createIterator.

private CloseableIterator<T> createIterator(final RocksDBClient client, final RangeReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final Collection<SinglePartitionQueryRanges> ranges, final Set<String> authorizations, final boolean async) {
    final Iterator<CloseableIterator> it = Arrays.stream(ArrayUtils.toObject(readerParams.getAdapterIds())).map(adapterId -> new RocksDBQueryExecution(client, RocksDBUtils.getTablePrefix(readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName()), adapterId, rowTransformer, ranges, new ClientVisibilityFilter(authorizations), DataStoreUtils.isMergingIteratorRequired(readerParams, client.isVisibilityEnabled()), async, RocksDBUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId), RocksDBUtils.isSortByKeyRequired(readerParams)).results()).iterator();
    final CloseableIterator<T>[] itArray = Iterators.toArray(it, CloseableIterator.class);
    return new CloseableIteratorWrapper<>(new Closeable() {

        AtomicBoolean closed = new AtomicBoolean(false);

        @Override
        public void close() throws IOException {
            if (!closed.getAndSet(true)) {
                Arrays.stream(itArray).forEach(it -> it.close());
            }
        }
    }, Iterators.concat(itArray));
}
Also used : Arrays(java.util.Arrays) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) RocksDBUtils(org.locationtech.geowave.datastore.rocksdb.util.RocksDBUtils) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) RocksDBClient(org.locationtech.geowave.datastore.rocksdb.util.RocksDBClient) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) RocksDBDataIndexTable(org.locationtech.geowave.datastore.rocksdb.util.RocksDBDataIndexTable) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) List(java.util.List) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) IOException(java.io.IOException) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter)

Example 2 with RocksDBClient

use of org.locationtech.geowave.datastore.rocksdb.util.RocksDBClient in project geowave by locationtech.

the class RocksDBReader method createIteratorForReader.

private CloseableIterator<T> createIteratorForReader(final RocksDBClient client, final ReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final boolean async) {
    final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges();
    final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations());
    if ((ranges != null) && !ranges.isEmpty()) {
        return createIterator(client, readerParams, readerParams.getRowTransformer(), ranges, authorizations, async);
    } else {
        final List<CloseableIterator<GeoWaveRow>> iterators = new ArrayList<>();
        for (final short adapterId : readerParams.getAdapterIds()) {
            final Pair<Boolean, Boolean> groupByRowAndSortByTime = RocksDBUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId);
            final String indexNamePrefix = RocksDBUtils.getTablePrefix(readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName());
            final Stream<CloseableIterator<GeoWaveRow>> streamIt = RocksDBUtils.getPartitions(client.getSubDirectory(), indexNamePrefix).stream().map(p -> RocksDBUtils.getIndexTableFromPrefix(client, indexNamePrefix, adapterId, p.getBytes(), groupByRowAndSortByTime.getRight()).iterator());
            iterators.addAll(streamIt.collect(Collectors.toList()));
        }
        return wrapResults(new Closeable() {

            AtomicBoolean closed = new AtomicBoolean(false);

            @Override
            public void close() throws IOException {
                if (!closed.getAndSet(true)) {
                    iterators.forEach(it -> it.close());
                }
            }
        }, Iterators.concat(iterators.iterator()), readerParams, rowTransformer, authorizations, client.isVisibilityEnabled());
    }
}
Also used : Arrays(java.util.Arrays) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) RocksDBUtils(org.locationtech.geowave.datastore.rocksdb.util.RocksDBUtils) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) RocksDBClient(org.locationtech.geowave.datastore.rocksdb.util.RocksDBClient) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) RocksDBDataIndexTable(org.locationtech.geowave.datastore.rocksdb.util.RocksDBDataIndexTable) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) List(java.util.List) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) ArrayList(java.util.ArrayList) IOException(java.io.IOException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean)

Aggregations

Iterators (com.google.common.collect.Iterators)2 Sets (com.google.common.collect.Sets)2 Streams (com.google.common.collect.Streams)2 Closeable (java.io.Closeable)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 Iterator (java.util.Iterator)2 List (java.util.List)2 Set (java.util.Set)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Collectors (java.util.stream.Collectors)2 Stream (java.util.stream.Stream)2 ArrayUtils (org.apache.commons.lang3.ArrayUtils)2 Pair (org.apache.commons.lang3.tuple.Pair)2 ByteArrayRange (org.locationtech.geowave.core.index.ByteArrayRange)2 SinglePartitionQueryRanges (org.locationtech.geowave.core.index.SinglePartitionQueryRanges)2 CloseableIterator (org.locationtech.geowave.core.store.CloseableIterator)2