use of org.locationtech.geowave.datastore.redis.util.GeoWaveRedisPersistedRow in project geowave by locationtech.
the class RedisReader method createIteratorForReader.
private CloseableIterator<T> createIteratorForReader(final RedissonClient client, final Compression compression, final ReaderParams<T> readerParams, final GeoWaveRowIteratorTransformer<T> rowTransformer, final String namespace, final boolean visibilityEnabled, final boolean async) {
final Collection<SinglePartitionQueryRanges> ranges = readerParams.getQueryRanges().getPartitionQueryRanges();
final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations());
if ((ranges != null) && !ranges.isEmpty()) {
return createIterator(client, compression, readerParams, readerParams.getRowTransformer(), namespace, ranges, authorizations, visibilityEnabled, async);
} else {
final Iterator<GeoWaveRedisRow>[] iterators = new Iterator[readerParams.getAdapterIds().length];
int i = 0;
for (final short adapterId : readerParams.getAdapterIds()) {
final Pair<Boolean, Boolean> groupByRowAndSortByTime = RedisUtils.isGroupByRowAndIsSortByTime(readerParams, adapterId);
final String setNamePrefix = RedisUtils.getRowSetPrefix(namespace, readerParams.getInternalAdapterStore().getTypeName(adapterId), readerParams.getIndex().getName());
final Stream<Pair<ByteArray, Iterator<ScoredEntry<GeoWaveRedisPersistedRow>>>> streamIt = RedisUtils.getPartitions(client, setNamePrefix).stream().map(p -> {
final Iterator<ScoredEntry<GeoWaveRedisPersistedRow>> result = RedisUtils.getRowSet(client, compression, setNamePrefix, p.getBytes(), groupByRowAndSortByTime.getRight(), visibilityEnabled).entryRange(Double.NEGATIVE_INFINITY, true, Double.POSITIVE_INFINITY, true);
final Iterator<ScoredEntry<GeoWaveRedisPersistedRow>> it = groupByRowAndSortByTime.getLeft() ? RedisUtils.groupByRow(result, groupByRowAndSortByTime.getRight()) : result;
return ImmutablePair.of(p, it);
});
iterators[i++] = Iterators.concat(streamIt.map(p -> Iterators.transform(p.getRight(), pr -> new GeoWaveRedisRow(pr.getValue(), adapterId, p.getLeft().getBytes(), RedisUtils.getFullSortKey(pr.getScore(), pr.getValue().getSortKeyPrecisionBeyondScore())))).iterator());
}
return wrapResults(Iterators.concat(iterators), readerParams, rowTransformer, authorizations, visibilityEnabled);
}
}
use of org.locationtech.geowave.datastore.redis.util.GeoWaveRedisPersistedRow in project geowave by locationtech.
the class BatchedRangeRead method executeQueryAsync.
private CloseableIterator<T> executeQueryAsync(final List<RangeReadInfo> reads) {
// first create a list of asynchronous query executions
final List<RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>>> futures = Lists.newArrayListWithExpectedSize(reads.size());
final BlockingQueue<Object> results = new LinkedBlockingQueue<>(MAX_BOUNDED_READS_ENQUEUED);
new Thread(new Runnable() {
@Override
public void run() {
// set it to 1 to make sure all queries are submitted in
// the loop
final AtomicInteger queryCount = new AtomicInteger(1);
for (final RangeReadInfo r : reads) {
try {
ByteArray partitionKey;
if ((r.partitionKey == null) || (r.partitionKey.length == 0)) {
partitionKey = EMPTY_PARTITION_KEY;
} else {
partitionKey = new ByteArray(r.partitionKey);
}
readSemaphore.acquire();
final RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>> f = setCache.get(partitionKey).entryRangeAsync(r.startScore, true, r.endScore, // sure the end is inclusive and do more precise client-side filtering
((r.endScore <= r.startScore) || (r.explicitEndCheck != null)));
queryCount.incrementAndGet();
f.handle((result, throwable) -> {
if (!f.isSuccess()) {
if (!f.isCancelled()) {
LOGGER.warn("Async Redis query failed", throwable);
}
checkFinalize(readSemaphore, results, queryCount);
return result;
} else {
try {
result.forEach(i -> i.getValue().setPartitionKey(r.partitionKey));
transformAndFilter(result.stream().filter(e -> r.passesExplicitRowChecks(e)).iterator()).forEachRemaining(row -> {
try {
results.put(row);
} catch (final InterruptedException e) {
LOGGER.warn("interrupted while waiting to enqueue a redis result", e);
}
});
} finally {
checkFinalize(readSemaphore, results, queryCount);
}
return result;
}
});
synchronized (futures) {
futures.add(f);
}
} catch (final InterruptedException e) {
LOGGER.warn("Exception while executing query", e);
readSemaphore.release();
}
}
// then decrement
if (queryCount.decrementAndGet() <= 0) {
// statements submitted
try {
results.put(RowConsumer.POISON);
} catch (final InterruptedException e) {
LOGGER.error("Interrupted while finishing blocking queue, this may result in deadlock!");
}
}
}
}, "Redis Query Executor").start();
return new CloseableIteratorWrapper<>(new Closeable() {
@Override
public void close() throws IOException {
List<RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>>> newFutures;
synchronized (futures) {
newFutures = new ArrayList<>(futures);
}
for (final RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>> f : newFutures) {
f.cancel(true);
}
}
}, new RowConsumer<>(results));
}
Aggregations