Search in sources :

Example 1 with RangeReaderParams

use of org.locationtech.geowave.core.store.operations.RangeReaderParams in project geowave by locationtech.

the class FileSystemReader method createIterator.

private CloseableIterator<T> createIterator(final FileSystemClient client, final RangeReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final Collection<SinglePartitionQueryRanges> ranges, final Set<String> authorizations, final boolean async) {
    final Iterator<CloseableIterator> it = Arrays.stream(ArrayUtils.toObject(readerParams.getAdapterIds())).map(adapterId -> new FileSystemQueryExecution(client, adapterId, readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName(), client.getFormat(), rowTransformer, ranges, new ClientVisibilityFilter(authorizations), DataStoreUtils.isMergingIteratorRequired(readerParams, client.isVisibilityEnabled()), async, FileSystemUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId), FileSystemUtils.isSortByKeyRequired(readerParams)).results()).iterator();
    final CloseableIterator<T>[] itArray = Iterators.toArray(it, CloseableIterator.class);
    return new CloseableIteratorWrapper<>(new Closeable() {

        AtomicBoolean closed = new AtomicBoolean(false);

        @Override
        public void close() throws IOException {
            if (!closed.getAndSet(true)) {
                Arrays.stream(itArray).forEach(it -> it.close());
            }
        }
    }, Iterators.concat(itArray));
}
Also used : Arrays(java.util.Arrays) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) IndexFormatter(org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) FileSystemDataIndexTable(org.locationtech.geowave.datastore.filesystem.util.FileSystemDataIndexTable) DataFormatterCache(org.locationtech.geowave.datastore.filesystem.util.DataFormatterCache) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) FileSystemUtils(org.locationtech.geowave.datastore.filesystem.util.FileSystemUtils) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) List(java.util.List) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) FileSystemClient(org.locationtech.geowave.datastore.filesystem.util.FileSystemClient) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) IOException(java.io.IOException) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter)

Example 2 with RangeReaderParams

use of org.locationtech.geowave.core.store.operations.RangeReaderParams in project geowave by locationtech.

the class RedisReader method createIterator.

private CloseableIterator<T> createIterator(final RedissonClient client, final Compression compression, final RangeReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final String namespace, final Collection<SinglePartitionQueryRanges> ranges, final Set<String> authorizations, final boolean visibilityEnabled, final boolean async) {
    final Iterator<CloseableIterator> it = Arrays.stream(ArrayUtils.toObject(readerParams.getAdapterIds())).map(adapterId -> new BatchedRangeRead(client, compression, RedisUtils.getRowSetPrefix(namespace, readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName()), adapterId, ranges, rowTransformer, new ClientVisibilityFilter(authorizations), DataStoreUtils.isMergingIteratorRequired(readerParams, visibilityEnabled), async, RedisUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId), RedisUtils.isSortByKeyRequired(readerParams), visibilityEnabled).results()).iterator();
    final CloseableIterator<T>[] itArray = Iterators.toArray(it, CloseableIterator.class);
    return new CloseableIteratorWrapper<>(new Closeable() {

        AtomicBoolean closed = new AtomicBoolean(false);

        @Override
        public void close() throws IOException {
            if (!closed.getAndSet(true)) {
                Arrays.stream(itArray).forEach(it -> it.close());
            }
        }
    }, Iterators.concat(itArray));
}
Also used : ByteArray(org.locationtech.geowave.core.index.ByteArray) Arrays(java.util.Arrays) GeoWaveRedisRow(org.locationtech.geowave.datastore.redis.util.GeoWaveRedisRow) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) RedisUtils(org.locationtech.geowave.datastore.redis.util.RedisUtils) Iterators(com.google.common.collect.Iterators) Compression(org.locationtech.geowave.datastore.redis.config.RedisOptions.Compression) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) RedissonClient(org.redisson.api.RedissonClient) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) ScoredEntry(org.redisson.client.protocol.ScoredEntry) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) GeoWaveRedisPersistedRow(org.locationtech.geowave.datastore.redis.util.GeoWaveRedisPersistedRow) Closeable(java.io.Closeable) Serialization(org.locationtech.geowave.datastore.redis.config.RedisOptions.Serialization) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) IOException(java.io.IOException) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter)

Example 3 with RangeReaderParams

use of org.locationtech.geowave.core.store.operations.RangeReaderParams in project geowave by locationtech.

the class RocksDBReader method createIterator.

private CloseableIterator<T> createIterator(final RocksDBClient client, final RangeReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final Collection<SinglePartitionQueryRanges> ranges, final Set<String> authorizations, final boolean async) {
    final Iterator<CloseableIterator> it = Arrays.stream(ArrayUtils.toObject(readerParams.getAdapterIds())).map(adapterId -> new RocksDBQueryExecution(client, RocksDBUtils.getTablePrefix(readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName()), adapterId, rowTransformer, ranges, new ClientVisibilityFilter(authorizations), DataStoreUtils.isMergingIteratorRequired(readerParams, client.isVisibilityEnabled()), async, RocksDBUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId), RocksDBUtils.isSortByKeyRequired(readerParams)).results()).iterator();
    final CloseableIterator<T>[] itArray = Iterators.toArray(it, CloseableIterator.class);
    return new CloseableIteratorWrapper<>(new Closeable() {

        AtomicBoolean closed = new AtomicBoolean(false);

        @Override
        public void close() throws IOException {
            if (!closed.getAndSet(true)) {
                Arrays.stream(itArray).forEach(it -> it.close());
            }
        }
    }, Iterators.concat(itArray));
}
Also used : Arrays(java.util.Arrays) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) RocksDBUtils(org.locationtech.geowave.datastore.rocksdb.util.RocksDBUtils) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) RocksDBClient(org.locationtech.geowave.datastore.rocksdb.util.RocksDBClient) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) RocksDBDataIndexTable(org.locationtech.geowave.datastore.rocksdb.util.RocksDBDataIndexTable) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) List(java.util.List) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) IOException(java.io.IOException) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter)

Aggregations

Iterators (com.google.common.collect.Iterators)3 Sets (com.google.common.collect.Sets)3 Streams (com.google.common.collect.Streams)3 Closeable (java.io.Closeable)3 IOException (java.io.IOException)3 Arrays (java.util.Arrays)3 Collection (java.util.Collection)3 Collections (java.util.Collections)3 Iterator (java.util.Iterator)3 Set (java.util.Set)3 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)3 Stream (java.util.stream.Stream)3 ArrayUtils (org.apache.commons.lang3.ArrayUtils)3 Pair (org.apache.commons.lang3.tuple.Pair)3 ByteArrayRange (org.locationtech.geowave.core.index.ByteArrayRange)3 SinglePartitionQueryRanges (org.locationtech.geowave.core.index.SinglePartitionQueryRanges)3 CloseableIterator (org.locationtech.geowave.core.store.CloseableIterator)3 CloseableIteratorWrapper (org.locationtech.geowave.core.store.CloseableIteratorWrapper)3 GeoWaveRow (org.locationtech.geowave.core.store.entities.GeoWaveRow)3 GeoWaveRowIteratorTransformer (org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer)3