Search in sources :

Example 1 with RowConsumer

use of org.locationtech.geowave.core.store.util.RowConsumer in project geowave by locationtech.

the class BatchedRangeRead method executeQueryAsync.

public CloseableIterator<T> executeQueryAsync(final Statement... statements) {
    // first create a list of asynchronous query executions
    final List<CompletionStage<AsyncResultSet>> futures = Lists.newArrayListWithExpectedSize(statements.length);
    final BlockingQueue<Object> results = new LinkedBlockingQueue<>(MAX_BOUNDED_READS_ENQUEUED);
    new Thread(new Runnable() {

        @Override
        public void run() {
            // set it to 1 to make sure all queries are submitted in
            // the loop
            final AtomicInteger queryCount = new AtomicInteger(1);
            for (final Statement s : statements) {
                try {
                    readSemaphore.acquire();
                    final CompletionStage<AsyncResultSet> f = operations.getSession().executeAsync(s);
                    synchronized (futures) {
                        futures.add(f);
                    }
                    queryCount.incrementAndGet();
                    f.whenCompleteAsync((result, t) -> {
                        if (result != null) {
                            try {
                                final Iterator<GeoWaveRow> iterator = (Iterator) Streams.stream(ResultSets.newInstance(result)).map(row -> new CassandraRow(row)).filter(filter).iterator();
                                rowTransformer.apply(rowMerging ? new GeoWaveRowMergingIterator(iterator) : iterator).forEachRemaining(row -> {
                                    try {
                                        results.put(row);
                                    } catch (final InterruptedException e) {
                                        LOGGER.warn("interrupted while waiting to enqueue a cassandra result", e);
                                    }
                                });
                            } finally {
                                checkFinalize(queryCount, results, readSemaphore);
                            }
                        } else if (t != null) {
                            checkFinalize(queryCount, results, readSemaphore);
                            // can do logging or start counting errors.
                            if (!(t instanceof CancellationException)) {
                                LOGGER.error("Failure from async query", t);
                                throw new RuntimeException(t);
                            }
                        }
                    });
                } catch (final InterruptedException e) {
                    LOGGER.warn("Exception while executing query", e);
                    readSemaphore.release();
                }
            }
            // then decrement
            if (queryCount.decrementAndGet() <= 0) {
                // statements submitted
                try {
                    results.put(RowConsumer.POISON);
                } catch (final InterruptedException e) {
                    LOGGER.error("Interrupted while finishing blocking queue, this may result in deadlock!");
                }
            }
        }
    }, "Cassandra Query Executor").start();
    return new CloseableIteratorWrapper<T>(new Closeable() {

        @Override
        public void close() throws IOException {
            synchronized (futures) {
                for (final CompletionStage<AsyncResultSet> f : futures) {
                    f.toCompletableFuture().cancel(true);
                }
            }
        }
    }, new RowConsumer(results));
}
Also used : CassandraRow(org.locationtech.geowave.datastore.cassandra.CassandraRow) Arrays(java.util.Arrays) ResultSets(com.datastax.oss.driver.internal.core.cql.ResultSets) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) LoggerFactory(org.slf4j.LoggerFactory) ArrayUtils(org.apache.commons.lang3.ArrayUtils) ByteBuffer(java.nio.ByteBuffer) ArrayList(java.util.ArrayList) AsyncResultSet(com.datastax.oss.driver.api.core.cql.AsyncResultSet) Lists(com.google.common.collect.Lists) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ByteArrayRange(org.locationtech.geowave.core.index.ByteArrayRange) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) CancellationException(java.util.concurrent.CancellationException) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) BoundStatementBuilder(com.datastax.oss.driver.api.core.cql.BoundStatementBuilder) Semaphore(java.util.concurrent.Semaphore) Predicate(java.util.function.Predicate) Collection(java.util.Collection) IOException(java.io.IOException) BlockingQueue(java.util.concurrent.BlockingQueue) BoundStatement(com.datastax.oss.driver.api.core.cql.BoundStatement) PreparedStatement(com.datastax.oss.driver.api.core.cql.PreparedStatement) Streams(com.google.common.collect.Streams) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) RowConsumer(org.locationtech.geowave.core.store.util.RowConsumer) TypeCodecs(com.datastax.oss.driver.api.core.type.codec.TypeCodecs) CassandraUtils(org.locationtech.geowave.datastore.cassandra.util.CassandraUtils) List(java.util.List) CompletionStage(java.util.concurrent.CompletionStage) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) Closeable(java.io.Closeable) CassandraField(org.locationtech.geowave.datastore.cassandra.CassandraRow.CassandraField) Statement(com.datastax.oss.driver.api.core.cql.Statement) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) AsyncResultSet(com.datastax.oss.driver.api.core.cql.AsyncResultSet) Closeable(java.io.Closeable) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Iterator(java.util.Iterator) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) CompletionStage(java.util.concurrent.CompletionStage) BoundStatement(com.datastax.oss.driver.api.core.cql.BoundStatement) PreparedStatement(com.datastax.oss.driver.api.core.cql.PreparedStatement) Statement(com.datastax.oss.driver.api.core.cql.Statement) CassandraRow(org.locationtech.geowave.datastore.cassandra.CassandraRow) IOException(java.io.IOException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CancellationException(java.util.concurrent.CancellationException) RowConsumer(org.locationtech.geowave.core.store.util.RowConsumer) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper)

Example 2 with RowConsumer

use of org.locationtech.geowave.core.store.util.RowConsumer in project geowave by locationtech.

the class KuduRangeRead method executeQueryAsync.

public CloseableIterator<T> executeQueryAsync(final List<AsyncKuduScanner> scanners) {
    final BlockingQueue<Object> results = new LinkedBlockingQueue<>(MAX_BOUNDED_READS_ENQUEUED);
    final AtomicBoolean isCanceled = new AtomicBoolean(false);
    new Thread(() -> {
        final AtomicInteger queryCount = new AtomicInteger(1);
        for (final AsyncKuduScanner scanner : scanners) {
            try {
                readSemaphore.acquire();
                executeScanner(scanner, readSemaphore, results, queryCount, isCanceled, visibilityEnabled, filter, rowTransformer, rowMerging);
            } catch (final InterruptedException e) {
                LOGGER.warn("Exception while executing query", e);
                readSemaphore.release();
            }
        }
        // then decrement
        if (queryCount.decrementAndGet() <= 0) {
            // and if there are no queries, there may not have been any statements submitted
            try {
                results.put(RowConsumer.POISON);
            } catch (final InterruptedException e) {
                LOGGER.error("Interrupted while finishing blocking queue, this may result in deadlock!");
            }
        }
    }, "Kudu Query Executor").start();
    return new CloseableIteratorWrapper<T>(() -> isCanceled.set(true), new RowConsumer(results));
}
Also used : AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AsyncKuduScanner(org.apache.kudu.client.AsyncKuduScanner) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RowConsumer(org.locationtech.geowave.core.store.util.RowConsumer) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper)

Example 3 with RowConsumer

use of org.locationtech.geowave.core.store.util.RowConsumer in project geowave by locationtech.

the class BatchedRangeRead method executeQueryAsync.

private CloseableIterator<T> executeQueryAsync(final List<RangeReadInfo> reads) {
    // first create a list of asynchronous query executions
    final List<RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>>> futures = Lists.newArrayListWithExpectedSize(reads.size());
    final BlockingQueue<Object> results = new LinkedBlockingQueue<>(MAX_BOUNDED_READS_ENQUEUED);
    new Thread(new Runnable() {

        @Override
        public void run() {
            // set it to 1 to make sure all queries are submitted in
            // the loop
            final AtomicInteger queryCount = new AtomicInteger(1);
            for (final RangeReadInfo r : reads) {
                try {
                    ByteArray partitionKey;
                    if ((r.partitionKey == null) || (r.partitionKey.length == 0)) {
                        partitionKey = EMPTY_PARTITION_KEY;
                    } else {
                        partitionKey = new ByteArray(r.partitionKey);
                    }
                    readSemaphore.acquire();
                    final RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>> f = setCache.get(partitionKey).entryRangeAsync(r.startScore, true, r.endScore, // sure the end is inclusive and do more precise client-side filtering
                    ((r.endScore <= r.startScore) || (r.explicitEndCheck != null)));
                    queryCount.incrementAndGet();
                    f.handle((result, throwable) -> {
                        if (!f.isSuccess()) {
                            if (!f.isCancelled()) {
                                LOGGER.warn("Async Redis query failed", throwable);
                            }
                            checkFinalize(readSemaphore, results, queryCount);
                            return result;
                        } else {
                            try {
                                result.forEach(i -> i.getValue().setPartitionKey(r.partitionKey));
                                transformAndFilter(result.stream().filter(e -> r.passesExplicitRowChecks(e)).iterator()).forEachRemaining(row -> {
                                    try {
                                        results.put(row);
                                    } catch (final InterruptedException e) {
                                        LOGGER.warn("interrupted while waiting to enqueue a redis result", e);
                                    }
                                });
                            } finally {
                                checkFinalize(readSemaphore, results, queryCount);
                            }
                            return result;
                        }
                    });
                    synchronized (futures) {
                        futures.add(f);
                    }
                } catch (final InterruptedException e) {
                    LOGGER.warn("Exception while executing query", e);
                    readSemaphore.release();
                }
            }
            // then decrement
            if (queryCount.decrementAndGet() <= 0) {
                // statements submitted
                try {
                    results.put(RowConsumer.POISON);
                } catch (final InterruptedException e) {
                    LOGGER.error("Interrupted while finishing blocking queue, this may result in deadlock!");
                }
            }
        }
    }, "Redis Query Executor").start();
    return new CloseableIteratorWrapper<>(new Closeable() {

        @Override
        public void close() throws IOException {
            List<RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>>> newFutures;
            synchronized (futures) {
                newFutures = new ArrayList<>(futures);
            }
            for (final RFuture<Collection<ScoredEntry<GeoWaveRedisPersistedRow>>> f : newFutures) {
                f.cancel(true);
            }
        }
    }, new RowConsumer<>(results));
}
Also used : ByteArray(org.locationtech.geowave.core.index.ByteArray) GeoWaveRedisRow(org.locationtech.geowave.datastore.redis.util.GeoWaveRedisRow) SinglePartitionQueryRanges(org.locationtech.geowave.core.index.SinglePartitionQueryRanges) GeoWaveRowIteratorTransformer(org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer) LoggerFactory(org.slf4j.LoggerFactory) RedisUtils(org.locationtech.geowave.datastore.redis.util.RedisUtils) Iterators(com.google.common.collect.Iterators) ArrayList(java.util.ArrayList) RFuture(org.redisson.api.RFuture) Compression(org.locationtech.geowave.datastore.redis.config.RedisOptions.Compression) Lists(com.google.common.collect.Lists) Pair(org.apache.commons.lang3.tuple.Pair) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RedissonClient(org.redisson.api.RedissonClient) GeoWaveRow(org.locationtech.geowave.core.store.entities.GeoWaveRow) UnsignedBytes(com.google.common.primitives.UnsignedBytes) RedisScoredSetWrapper(org.locationtech.geowave.datastore.redis.util.RedisScoredSetWrapper) Caffeine(com.github.benmanes.caffeine.cache.Caffeine) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) GeoWaveRowMergingIterator(org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator) LoadingCache(com.github.benmanes.caffeine.cache.LoadingCache) Semaphore(java.util.concurrent.Semaphore) Predicate(java.util.function.Predicate) Collection(java.util.Collection) IOException(java.io.IOException) BlockingQueue(java.util.concurrent.BlockingQueue) Streams(com.google.common.collect.Streams) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) RowConsumer(org.locationtech.geowave.core.store.util.RowConsumer) ScoredEntry(org.redisson.client.protocol.ScoredEntry) Serializable(java.io.Serializable) List(java.util.List) CloseableIterator(org.locationtech.geowave.core.store.CloseableIterator) GeoWaveRedisPersistedRow(org.locationtech.geowave.datastore.redis.util.GeoWaveRedisPersistedRow) Closeable(java.io.Closeable) Comparator(java.util.Comparator) Closeable(java.io.Closeable) ArrayList(java.util.ArrayList) IOException(java.io.IOException) RFuture(org.redisson.api.RFuture) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) GeoWaveRedisPersistedRow(org.locationtech.geowave.datastore.redis.util.GeoWaveRedisPersistedRow) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ScoredEntry(org.redisson.client.protocol.ScoredEntry) ByteArray(org.locationtech.geowave.core.index.ByteArray) Collection(java.util.Collection) ArrayList(java.util.ArrayList) List(java.util.List) CloseableIteratorWrapper(org.locationtech.geowave.core.store.CloseableIteratorWrapper)

Aggregations

LinkedBlockingQueue (java.util.concurrent.LinkedBlockingQueue)3 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)3 CloseableIteratorWrapper (org.locationtech.geowave.core.store.CloseableIteratorWrapper)3 RowConsumer (org.locationtech.geowave.core.store.util.RowConsumer)3 Lists (com.google.common.collect.Lists)2 Streams (com.google.common.collect.Streams)2 Closeable (java.io.Closeable)2 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 Collection (java.util.Collection)2 Iterator (java.util.Iterator)2 List (java.util.List)2 BlockingQueue (java.util.concurrent.BlockingQueue)2 Semaphore (java.util.concurrent.Semaphore)2 Predicate (java.util.function.Predicate)2 SinglePartitionQueryRanges (org.locationtech.geowave.core.index.SinglePartitionQueryRanges)2 CloseableIterator (org.locationtech.geowave.core.store.CloseableIterator)2 GeoWaveRow (org.locationtech.geowave.core.store.entities.GeoWaveRow)2 GeoWaveRowIteratorTransformer (org.locationtech.geowave.core.store.entities.GeoWaveRowIteratorTransformer)2 GeoWaveRowMergingIterator (org.locationtech.geowave.core.store.entities.GeoWaveRowMergingIterator)2