Search in sources :

Example 1 with FileSystemClient

use of org.locationtech.geowave.datastore.filesystem.util.FileSystemClient in project geowave by locationtech.

the class FileSystemReader method createIterator.

private CloseableIterator<T> createIterator(final FileSystemClient client, final RangeReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final Collection<SinglePartitionQueryRanges> ranges, final Set<String> authorizations, final boolean async) {
    final Iterator<CloseableIterator> it = Arrays.stream(ArrayUtils.toObject(readerParams.getAdapterIds())).map(adapterId -> new FileSystemQueryExecution(client, adapterId, readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName(), client.getFormat(), rowTransformer, ranges, new ClientVisibilityFilter(authorizations), DataStoreUtils.isMergingIteratorRequired(readerParams, client.isVisibilityEnabled()), async, FileSystemUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId), FileSystemUtils.isSortByKeyRequired(readerParams)).results()).iterator();
    final CloseableIterator<T>[] itArray = Iterators.toArray(it, CloseableIterator.class);
    return new CloseableIteratorWrapper<>(new Closeable() {

        AtomicBoolean closed = new AtomicBoolean(false);

        @Override
        public void close() throws IOException {
            if (!closed.getAndSet(true)) {
                Arrays.stream(itArray).forEach(it -> it.close());
            }
        }
    }, Iterators.concat(itArray));
}
Also used : Arrays(java.util.Arrays) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) IndexFormatter(org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) FileSystemDataIndexTable(org.locationtech.geowave.datastore.filesystem.util.FileSystemDataIndexTable) DataFormatterCache(org.locationtech.geowave.datastore.filesystem.util.DataFormatterCache) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) FileSystemUtils(org.locationtech.geowave.datastore.filesystem.util.FileSystemUtils) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) List(java.util.List) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) FileSystemClient(org.locationtech.geowave.datastore.filesystem.util.FileSystemClient) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) IOException(java.io.IOException) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter)

Example 2 with FileSystemClient

use of org.locationtech.geowave.datastore.filesystem.util.FileSystemClient in project geowave by locationtech.

the class FileSystemReader method createIteratorForReader.

private CloseableIterator<T> createIteratorForReader(final FileSystemClient client, final ReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final boolean async) {
    final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges();
    final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations());
    if ((ranges != null) && !ranges.isEmpty()) {
        return createIterator(client, readerParams, readerParams.getRowTransformer(), ranges, authorizations, async);
    } else {
        final List<CloseableIterator<GeoWaveRow>> iterators = new ArrayList<>();
        final IndexFormatter indexFormatter = DataFormatterCache.getInstance().getFormatter(client.getFormat(), client.isVisibilityEnabled()).getIndexFormatter();
        final String indexName = readerParams.getIndex().getName();
        for (final short adapterId : readerParams.getAdapterIds()) {
            final Pair<Boolean, Boolean> groupByRowAndSortByTime = FileSystemUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId);
            final String typeName = readerParams.getInternalAdapterStore().getTypeName(adapterId);
            final String indexDirectory = indexFormatter.getDirectoryName(indexName, typeName);
            final Stream<CloseableIterator<GeoWaveRow>> streamIt = FileSystemUtils.getPartitions(FileSystemUtils.getSubdirectory(client.getSubDirectory(), indexDirectory), indexFormatter, indexName, typeName).stream().map(p -> FileSystemUtils.getIndexTable(client, adapterId, typeName, indexName, p.getBytes(), groupByRowAndSortByTime.getRight()).iterator());
            iterators.addAll(streamIt.collect(Collectors.toList()));
        }
        return wrapResults(new Closeable() {

            AtomicBoolean closed = new AtomicBoolean(false);

            @Override
            public void close() throws IOException {
                if (!closed.getAndSet(true)) {
                    iterators.forEach(it -> it.close());
                }
            }
        }, Iterators.concat(iterators.iterator()), readerParams, rowTransformer, authorizations, client.isVisibilityEnabled());
    }
}
Also used : Arrays(java.util.Arrays) RecordReaderParams(org.locationtech.geowave.mapreduce.splits.RecordReaderParams) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ArrayUtils(org.apache.commons.lang3.ArrayUtils) IndexFormatter(org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) ReaderParams(org.locationtech.geowave.core.store.operations.ReaderParams) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) ClientVisibilityFilter(org.locationtech.geowave.core.store.query.filter.ClientVisibilityFilter) FileSystemDataIndexTable(org.locationtech.geowave.datastore.filesystem.util.FileSystemDataIndexTable) DataFormatterCache(org.locationtech.geowave.datastore.filesystem.util.DataFormatterCache) DataStoreUtils(org.locationtech.geowave.core.store.util.DataStoreUtils) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Iterator(java.util.Iterator) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) DataIndexReaderParams(org.locationtech.geowave.core.store.operations.DataIndexReaderParams) RowReader(org.locationtech.geowave.core.store.operations.RowReader) Collection(java.util.Collection) Set(java.util.Set) FileSystemUtils(org.locationtech.geowave.datastore.filesystem.util.FileSystemUtils) IOException(java.io.IOException) Streams(com.google.common.collect.Streams) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) RangeReaderParams(org.locationtech.geowave.core.store.operations.RangeReaderParams) List(java.util.List) Stream(java.util.stream.Stream) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) GeoWaveRowRange(org.locationtech.geowave.mapreduce.splits.GeoWaveRowRange) Collections(java.util.Collections) FileSystemClient(org.locationtech.geowave.datastore.filesystem.util.FileSystemClient) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) ArrayList(java.util.ArrayList) IOException(java.io.IOException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IndexFormatter(org.locationtech.geowave.datastore.filesystem.FileSystemDataFormatter.IndexFormatter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean)

Aggregations

Iterators (com.google.common.collect.Iterators)2 Sets (com.google.common.collect.Sets)2 Streams (com.google.common.collect.Streams)2 Closeable (java.io.Closeable)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Arrays (java.util.Arrays)2 Collection (java.util.Collection)2 Collections (java.util.Collections)2 Iterator (java.util.Iterator)2 List (java.util.List)2 Set (java.util.Set)2 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)2 Collectors (java.util.stream.Collectors)2 Stream (java.util.stream.Stream)2 ArrayUtils (org.apache.commons.lang3.ArrayUtils)2 Pair (org.apache.commons.lang3.tuple.Pair)2 ByteArrayRange (org.locationtech.geowave.core.index.ByteArrayRange)2 SinglePartitionQueryRanges (org.locationtech.geowave.core.index.SinglePartitionQueryRanges)2 CloseableIterator (org.locationtech.geowave.core.store.CloseableIterator)2