use of org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator in project geowave by locationtech.
the class CassandraReader method wrapResults.
@SuppressWarnings("unchecked")
private CloseableIterator<T> wrapResults(final CloseableIterator<CassandraRow> results, final RangeReaderParams<T> readerParams) {
final Set<String> authorizations = Sets.newHashSet(readerParams.getAdditionalAuthorizations());
final Iterator<GeoWaveRow> iterator = (Iterator) Streams.stream(results).filter(new ClientVisibilityFilter(authorizations)).iterator();
return new CloseableIteratorWrapper<>(results, rowTransformer.apply(DataStoreUtils.isMergingIteratorRequired(readerParams, visibilityEnabled) ? new GeoWaveRowMergingIterator(iterator) : iterator));
}
use of org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator in project geowave by locationtech.
the class BatchedRangeRead method executeQueryAsync.
public CloseableIterator<T> executeQueryAsync(final Statement... statements) {
// first create a list of asynchronous query executions
final List<CompletionStage<AsyncResultSet>> futures = Lists.newArrayListWithExpectedSize(statements.length);
final BlockingQueue<Object> results = new LinkedBlockingQueue<>(MAX_BOUNDED_READS_ENQUEUED);
new Thread(new Runnable() {
@Override
public void run() {
// set it to 1 to make sure all queries are submitted in
// the loop
final AtomicInteger queryCount = new AtomicInteger(1);
for (final Statement s : statements) {
try {
readSemaphore.acquire();
final CompletionStage<AsyncResultSet> f = operations.getSession().executeAsync(s);
synchronized (futures) {
futures.add(f);
}
queryCount.incrementAndGet();
f.whenCompleteAsync((result, t) -> {
if (result != null) {
try {
final Iterator<GeoWaveRow> iterator = (Iterator) Streams.stream(ResultSets.newInstance(result)).map(row -> new CassandraRow(row)).filter(filter).iterator();
rowTransformer.apply(rowMerging ? new GeoWaveRowMergingIterator(iterator) : iterator).forEachRemaining(row -> {
try {
results.put(row);
} catch (final InterruptedException e) {
LOGGER.warn("interrupted while waiting to enqueue a cassandra result", e);
}
});
} finally {
checkFinalize(queryCount, results, readSemaphore);
}
} else if (t != null) {
checkFinalize(queryCount, results, readSemaphore);
// can do logging or start counting errors.
if (!(t instanceof CancellationException)) {
LOGGER.error("Failure from async query", t);
throw new RuntimeException(t);
}
}
});
} catch (final InterruptedException e) {
LOGGER.warn("Exception while executing query", e);
readSemaphore.release();
}
}
// then decrement
if (queryCount.decrementAndGet() <= 0) {
// statements submitted
try {
results.put(RowConsumer.POISON);
} catch (final InterruptedException e) {
LOGGER.error("Interrupted while finishing blocking queue, this may result in deadlock!");
}
}
}
}, "Cassandra Query Executor").start();
return new CloseableIteratorWrapper<T>(new Closeable() {
@Override
public void close() throws IOException {
synchronized (futures) {
for (final CompletionStage<AsyncResultSet> f : futures) {
f.toCompletableFuture().cancel(true);
}
}
}
}, new RowConsumer(results));
}
use of org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator in project geowave by locationtech.
the class KuduRangeRead method executeScanner.
public Deferred<Object> executeScanner(final AsyncKuduScanner scanner, final Semaphore semaphore, final BlockingQueue<Object> resultQueue, final AtomicInteger queryCount, final AtomicBoolean isCanceled, final boolean visibilityEnabled, final Predicate<GeoWaveRow> filter, final GeoWaveRowIteratorTransformer<T> rowTransformer, final boolean rowMerging) {
// Errback class
class QueryErrback implements Callback<Deferred<Object>, Exception> {
@Override
public Deferred<Object> call(final Exception e) {
LOGGER.warn("While scanning rows from kudu", e);
checkFinalize(scanner, semaphore, resultQueue, queryCount);
return Deferred.fromError(e);
}
}
final QueryErrback errBack = new QueryErrback();
// callback class
class QueryCallback implements Callback<Deferred<Object>, RowResultIterator> {
@Override
public Deferred<Object> call(final RowResultIterator rs) {
if ((rs == null) || isCanceled.get()) {
checkFinalize(scanner, semaphore, resultQueue, queryCount);
return Deferred.fromResult(null);
}
if (rs.getNumRows() > 0) {
Stream<GeoWaveRow> tmpStream = Streams.stream(rs.iterator()).map(KuduRow::new);
if (visibilityEnabled) {
tmpStream = tmpStream.filter(filter);
}
final Iterator<GeoWaveRow> tmpIterator = tmpStream.iterator();
rowTransformer.apply(rowMerging ? new GeoWaveRowMergingIterator(tmpIterator) : tmpIterator).forEachRemaining(row -> {
try {
resultQueue.put(row);
} catch (final InterruptedException e) {
LOGGER.warn("interrupted while waiting to enqueue a kudu result", e);
}
});
}
if (scanner.hasMoreRows()) {
return scanner.nextRows().addCallbackDeferring(this).addErrback(errBack);
}
checkFinalize(scanner, semaphore, resultQueue, queryCount);
return Deferred.fromResult(null);
}
}
queryCount.incrementAndGet();
return scanner.nextRows().addCallbackDeferring(new QueryCallback()).addErrback(errBack);
}
use of org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator in project geowave by locationtech.
the class DynamoDBReader method startRead.
private void startRead(final List<QueryRequest> requests, final String tableName, final boolean rowMerging, final boolean parallelDecode) {
Iterator<Map<String, AttributeValue>> rawIterator;
Predicate<DynamoDBRow> adapterIdFilter = null;
final Function<Iterator<Map<String, AttributeValue>>, Iterator<DynamoDBRow>> rawToDynamoDBRow = new Function<Iterator<Map<String, AttributeValue>>, Iterator<DynamoDBRow>>() {
@Override
public Iterator<DynamoDBRow> apply(final Iterator<Map<String, AttributeValue>> input) {
final Iterator<DynamoDBRow> rowIterator = Streams.stream(input).map(new DynamoDBRow.GuavaRowTranslationHelper()).filter(visibilityFilter).iterator();
if (rowMerging) {
return new GeoWaveRowMergingIterator<>(rowIterator);
} else {
// TODO: understand why there are duplicates coming back when there shouldn't be from
// DynamoDB
final DedupeFilter dedupe = new DedupeFilter();
return Iterators.filter(rowIterator, row -> dedupe.applyDedupeFilter(row.getAdapterId(), new ByteArray(row.getDataId())));
}
}
};
if (!requests.isEmpty()) {
if (ASYNC) {
rawIterator = Iterators.concat(requests.parallelStream().map(this::executeAsyncQueryRequest).iterator());
} else {
rawIterator = Iterators.concat(requests.parallelStream().map(this::executeQueryRequest).iterator());
}
} else {
if (ASYNC) {
final ScanRequest request = new ScanRequest(tableName);
rawIterator = new AsyncPaginatedScan(request, operations.getClient());
} else {
// query everything
final ScanRequest request = new ScanRequest(tableName);
final ScanResult scanResult = operations.getClient().scan(request);
rawIterator = new LazyPaginatedScan(scanResult, request, operations.getClient());
// filtering by adapter ID
if ((readerParams.getAdapterIds() != null) && (readerParams.getAdapterIds().length > 0)) {
adapterIdFilter = input -> ArrayUtils.contains(readerParams.getAdapterIds(), input.getAdapterId());
}
}
}
Iterator<DynamoDBRow> rowIter = rawToDynamoDBRow.apply(rawIterator);
if (adapterIdFilter != null) {
rowIter = Streams.stream(rowIter).filter(adapterIdFilter).iterator();
}
if (parallelDecode) {
final ParallelDecoder<T> decoder = new SimpleParallelDecoder<>(rowTransformer, Iterators.transform(rowIter, r -> (GeoWaveRow) r));
try {
decoder.startDecode();
} catch (final Exception e) {
Throwables.propagate(e);
}
iterator = decoder;
closeable = decoder;
} else {
iterator = rowTransformer.apply(Iterators.transform(rowIter, r -> (GeoWaveRow) r));
closeable = null;
}
}
Aggregations