Examples with LazyPaginatedScan org.locationtech.geowave.datastore.dynamodb.util.LazyPaginatedScan used on opensource projects

Example 1 with LazyPaginatedScan

use of org.locationtech.geowave.datastore.dynamodb.util.LazyPaginatedScan in project geowave by locationtech.

the class DynamoDBReader method startRead.

private void startRead(final List<QueryRequest> requests, final String tableName, final boolean rowMerging, final boolean parallelDecode) {
    Iterator<Map<String, AttributeValue>> rawIterator;
    Predicate<DynamoDBRow> adapterIdFilter = null;
    final Function<Iterator<Map<String, AttributeValue>>, Iterator<DynamoDBRow>> rawToDynamoDBRow = new Function<Iterator<Map<String, AttributeValue>>, Iterator<DynamoDBRow>>() {

        @Override
        public Iterator<DynamoDBRow> apply(final Iterator<Map<String, AttributeValue>> input) {
            final Iterator<DynamoDBRow> rowIterator = Streams.stream(input).map(new DynamoDBRow.GuavaRowTranslationHelper()).filter(visibilityFilter).iterator();
            if (rowMerging) {
                return new GeoWaveRowMergingIterator<>(rowIterator);
            } else {
                // TODO: understand why there are duplicates coming back when there shouldn't be from
                // DynamoDB
                final DedupeFilter dedupe = new DedupeFilter();
                return Iterators.filter(rowIterator, row -> dedupe.applyDedupeFilter(row.getAdapterId(), new ByteArray(row.getDataId())));
            }
        }
    };
    if (!requests.isEmpty()) {
        if (ASYNC) {
            rawIterator = Iterators.concat(requests.parallelStream().map(this::executeAsyncQueryRequest).iterator());
        } else {
            rawIterator = Iterators.concat(requests.parallelStream().map(this::executeQueryRequest).iterator());
        }
    } else {
        if (ASYNC) {
            final ScanRequest request = new ScanRequest(tableName);
            rawIterator = new AsyncPaginatedScan(request, operations.getClient());
        } else {
            // query everything
            final ScanRequest request = new ScanRequest(tableName);
            final ScanResult scanResult = operations.getClient().scan(request);
            rawIterator = new LazyPaginatedScan(scanResult, request, operations.getClient());
            // filtering by adapter ID
            if ((readerParams.getAdapterIds() != null) && (readerParams.getAdapterIds().length > 0)) {
                adapterIdFilter = input -> ArrayUtils.contains(readerParams.getAdapterIds(), input.getAdapterId());
            }
        }
    }
    Iterator<DynamoDBRow> rowIter = rawToDynamoDBRow.apply(rawIterator);
    if (adapterIdFilter != null) {
        rowIter = Streams.stream(rowIter).filter(adapterIdFilter).iterator();
    }
    if (parallelDecode) {
        final ParallelDecoder<T> decoder = new SimpleParallelDecoder<>(rowTransformer, Iterators.transform(rowIter, r -> (GeoWaveRow) r));
        try {
            decoder.startDecode();
        } catch (final Exception e) {
            Throwables.propagate(e);
        }
        iterator = decoder;
        closeable = decoder;
    } else {
        iterator = rowTransformer.apply(Iterators.transform(rowIter, r -> (GeoWaveRow) r));
        closeable = null;
    }
}

Example 2 with LazyPaginatedScan

use of org.locationtech.geowave.datastore.dynamodb.util.LazyPaginatedScan in project geowave by locationtech.

the class DynamoDBMetadataReader method query.

@Override
public CloseableIterator<GeoWaveMetadata> query(final MetadataQuery query) {
    final String tableName = operations.getMetadataTableName(metadataType);
    final boolean needsVisibility = metadataType.isStatValues() && operations.getOptions().getBaseOptions().isVisibilityEnabled();
    final Iterator<Map<String, AttributeValue>> iterator;
    if (!query.hasPrimaryIdRanges()) {
        if (query.hasPrimaryId() && query.isExact()) {
            final QueryRequest queryRequest = new QueryRequest(tableName);
            if (query.hasSecondaryId()) {
                queryRequest.withFilterExpression(DynamoDBOperations.METADATA_SECONDARY_ID_KEY + " = :secVal").addExpressionAttributeValuesEntry(":secVal", new AttributeValue().withB(ByteBuffer.wrap(query.getSecondaryId())));
            }
            queryRequest.withKeyConditionExpression(DynamoDBOperations.METADATA_PRIMARY_ID_KEY + " = :priVal").addExpressionAttributeValuesEntry(":priVal", new AttributeValue().withB(ByteBuffer.wrap(query.getPrimaryId())));
            final QueryResult queryResult = operations.getClient().query(queryRequest);
            return wrapIterator(queryResult.getItems().iterator(), query, needsVisibility);
        }
        final ScanRequest scan = new ScanRequest(tableName);
        if (query.hasPrimaryId()) {
            scan.addScanFilterEntry(DynamoDBOperations.METADATA_PRIMARY_ID_KEY, new Condition().withAttributeValueList(new AttributeValue().withB(ByteBuffer.wrap(query.getPrimaryId()))).withComparisonOperator(ComparisonOperator.BEGINS_WITH));
        }
        if (query.hasSecondaryId()) {
            scan.addScanFilterEntry(DynamoDBOperations.METADATA_SECONDARY_ID_KEY, new Condition().withAttributeValueList(new AttributeValue().withB(ByteBuffer.wrap(query.getSecondaryId()))).withComparisonOperator(ComparisonOperator.EQ));
        }
        final ScanResult scanResult = operations.getClient().scan(scan);
        iterator = new LazyPaginatedScan(scanResult, scan, operations.getClient());
    } else {
        iterator = Iterators.concat(Arrays.stream(query.getPrimaryIdRanges()).map(r -> {
            final ScanRequest scan = new ScanRequest(tableName);
            if (query.hasSecondaryId()) {
                scan.addScanFilterEntry(DynamoDBOperations.METADATA_SECONDARY_ID_KEY, new Condition().withAttributeValueList(new AttributeValue().withB(ByteBuffer.wrap(query.getSecondaryId()))).withComparisonOperator(ComparisonOperator.EQ));
            }
            if (r.getStart() != null) {
                if (r.getEnd() != null) {
                    scan.addScanFilterEntry(DynamoDBOperations.METADATA_PRIMARY_ID_KEY, new Condition().withAttributeValueList(new AttributeValue().withB(ByteBuffer.wrap(r.getStart())), new AttributeValue().withB(ByteBuffer.wrap(ByteArrayUtils.getNextInclusive(r.getEnd())))).withComparisonOperator(ComparisonOperator.BETWEEN));
                } else {
                    scan.addScanFilterEntry(DynamoDBOperations.METADATA_PRIMARY_ID_KEY, new Condition().withAttributeValueList(new AttributeValue().withB(ByteBuffer.wrap(r.getStart()))).withComparisonOperator(ComparisonOperator.GE));
                }
            } else if (r.getEnd() != null) {
                scan.addScanFilterEntry(DynamoDBOperations.METADATA_PRIMARY_ID_KEY, new Condition().withAttributeValueList(new AttributeValue().withB(ByteBuffer.wrap(r.getEndAsNextPrefix()))).withComparisonOperator(ComparisonOperator.LT));
            }
            final ScanResult scanResult = operations.getClient().scan(scan);
            return new LazyPaginatedScan(scanResult, scan, operations.getClient());
        }).iterator());
    }
    return wrapIterator(iterator, query, needsVisibility);
}