Examples with ParallelDecoder org.locationtech.geowave.core.store.operations.ParallelDecoder used on opensource projects

Example 1 with ParallelDecoder

use of org.locationtech.geowave.core.store.operations.ParallelDecoder in project geowave by locationtech.

the class DynamoDBReader method startRead.

private void startRead(final List<QueryRequest> requests, final String tableName, final boolean rowMerging, final boolean parallelDecode) {
    Iterator<Map<String, AttributeValue>> rawIterator;
    Predicate<DynamoDBRow> adapterIdFilter = null;
    final Function<Iterator<Map<String, AttributeValue>>, Iterator<DynamoDBRow>> rawToDynamoDBRow = new Function<Iterator<Map<String, AttributeValue>>, Iterator<DynamoDBRow>>() {

        @Override
        public Iterator<DynamoDBRow> apply(final Iterator<Map<String, AttributeValue>> input) {
            final Iterator<DynamoDBRow> rowIterator = Streams.stream(input).map(new DynamoDBRow.GuavaRowTranslationHelper()).filter(visibilityFilter).iterator();
            if (rowMerging) {
                return new GeoWaveRowMergingIterator<>(rowIterator);
            } else {
                // TODO: understand why there are duplicates coming back when there shouldn't be from
                // DynamoDB
                final DedupeFilter dedupe = new DedupeFilter();
                return Iterators.filter(rowIterator, row -> dedupe.applyDedupeFilter(row.getAdapterId(), new ByteArray(row.getDataId())));
            }
        }
    };
    if (!requests.isEmpty()) {
        if (ASYNC) {
            rawIterator = Iterators.concat(requests.parallelStream().map(this::executeAsyncQueryRequest).iterator());
        } else {
            rawIterator = Iterators.concat(requests.parallelStream().map(this::executeQueryRequest).iterator());
        }
    } else {
        if (ASYNC) {
            final ScanRequest request = new ScanRequest(tableName);
            rawIterator = new AsyncPaginatedScan(request, operations.getClient());
        } else {
            // query everything
            final ScanRequest request = new ScanRequest(tableName);
            final ScanResult scanResult = operations.getClient().scan(request);
            rawIterator = new LazyPaginatedScan(scanResult, request, operations.getClient());
            // filtering by adapter ID
            if ((readerParams.getAdapterIds() != null) && (readerParams.getAdapterIds().length > 0)) {
                adapterIdFilter = input -> ArrayUtils.contains(readerParams.getAdapterIds(), input.getAdapterId());
            }
        }
    }
    Iterator<DynamoDBRow> rowIter = rawToDynamoDBRow.apply(rawIterator);
    if (adapterIdFilter != null) {
        rowIter = Streams.stream(rowIter).filter(adapterIdFilter).iterator();
    }
    if (parallelDecode) {
        final ParallelDecoder<T> decoder = new SimpleParallelDecoder<>(rowTransformer, Iterators.transform(rowIter, r -> (GeoWaveRow) r));
        try {
            decoder.startDecode();
        } catch (final Exception e) {
            Throwables.propagate(e);
        }
        iterator = decoder;
        closeable = decoder;
    } else {
        iterator = rowTransformer.apply(Iterators.transform(rowIter, r -> (GeoWaveRow) r));
        closeable = null;
    }
}