Examples with AsyncIterator io.pravega.common.util.AsyncIterator used on opensource projects

Example 1 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class ControllerImpl method listStreams.

@Override
public AsyncIterator<Stream> listStreams(String scopeName) {
    Exceptions.checkNotClosed(closed.get(), this);
    long traceId = LoggerHelpers.traceEnter(log, "listStreams", scopeName);
    long requestId = requestIdGenerator.get();
    try {
        final Function<ContinuationToken, CompletableFuture<Map.Entry<ContinuationToken, Collection<Stream>>>> function = token -> this.retryConfig.runAsync(() -> {
            RPCAsyncCallback<StreamsInScopeResponse> callback = new RPCAsyncCallback<>(requestId, "listStreams", scopeName);
            ScopeInfo scopeInfo = ScopeInfo.newBuilder().setScope(scopeName).build();
            new ControllerClientTagger(client, timeoutMillis).withTag(requestId, LIST_STREAMS_IN_SCOPE, scopeName).listStreamsInScope(StreamsInScopeRequest.newBuilder().setScope(scopeInfo).setContinuationToken(token).build(), callback);
            return callback.getFuture().thenApplyAsync(x -> {
                switch(x.getStatus()) {
                    case SCOPE_NOT_FOUND:
                        log.warn(requestId, "Scope not found: {}", scopeName);
                        throw new NoSuchScopeException();
                    case FAILURE:
                        log.warn(requestId, "Internal Server Error while trying to list streams in scope: {}", scopeName);
                        throw new RuntimeException("Failure while trying to list streams");
                    case SUCCESS:
                    // compatibility reasons
                    default:
                        List<Stream> result = x.getStreamsList().stream().map(y -> new StreamImpl(y.getScope(), y.getStream())).collect(Collectors.toList());
                        return new AbstractMap.SimpleEntry<>(x.getContinuationToken(), result);
                }
            }, this.executor);
        }, this.executor);
        return new ContinuationTokenAsyncIterator<>(function, ContinuationToken.newBuilder().build());
    } finally {
        LoggerHelpers.traceLeave(log, "listStreams", traceId);
    }
}

Example 2 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class ControllerImpl method listStreamsForTag.

@Override
public AsyncIterator<Stream> listStreamsForTag(String scopeName, String tag) {
    Exceptions.checkNotClosed(closed.get(), this);
    long traceId = LoggerHelpers.traceEnter(log, LIST_STREAMS_IN_SCOPE_FOR_TAG, scopeName);
    long requestId = requestIdGenerator.get();
    try {
        final Function<ContinuationToken, CompletableFuture<Map.Entry<ContinuationToken, Collection<Stream>>>> function = token -> this.retryConfig.runAsync(() -> {
            RPCAsyncCallback<StreamsInScopeResponse> callback = new RPCAsyncCallback<>(requestId, LIST_STREAMS_IN_SCOPE_FOR_TAG, scopeName);
            ScopeInfo scopeInfo = ScopeInfo.newBuilder().setScope(scopeName).build();
            StreamsInScopeWithTagRequest request = StreamsInScopeWithTagRequest.newBuilder().setScope(scopeInfo).setContinuationToken(token).setTag(tag).build();
            new ControllerClientTagger(client, timeoutMillis).withTag(requestId, LIST_STREAMS_IN_SCOPE_FOR_TAG, scopeName).listStreamsInScopeForTag(request, callback);
            return callback.getFuture().thenApplyAsync(x -> {
                switch(x.getStatus()) {
                    case SCOPE_NOT_FOUND:
                        log.warn(requestId, "Scope not found: {}", scopeName);
                        throw new NoSuchScopeException();
                    case FAILURE:
                        log.warn(requestId, "Internal Server Error while trying to list streams in scope: {} with tag: {}", scopeName, tag);
                        throw new RuntimeException("Failure while trying to list streams with tag");
                    case SUCCESS:
                    // compatibility reasons
                    default:
                        List<Stream> result = x.getStreamsList().stream().map(y -> new StreamImpl(y.getScope(), y.getStream())).collect(Collectors.toList());
                        return new AbstractMap.SimpleEntry<>(x.getContinuationToken(), result);
                }
            }, this.executor);
        }, this.executor);
        return new ContinuationTokenAsyncIterator<>(function, ContinuationToken.newBuilder().build());
    } finally {
        LoggerHelpers.traceLeave(log, LIST_STREAMS_IN_SCOPE_FOR_TAG, traceId);
    }
}

Example 3 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class HashTableSegmentLayout method newIterator.

private <T> CompletableFuture<AsyncIterator<IteratorItem<T>>> newIterator(@NonNull DirectSegmentAccess segment, @NonNull IteratorArgs args, @NonNull GetBucketReader<T> createBucketReader) {
    Preconditions.checkArgument(args.getFrom() == null && args.getTo() == null, "Range Iterators not supported for HashTableSegments.");
    UUID fromHash;
    BufferView serializedState = args.getContinuationToken();
    try {
        fromHash = KeyHasher.getNextHash(serializedState == null ? null : IteratorStateImpl.deserialize(serializedState).getKeyHash());
    } catch (IOException ex) {
        // Bad IteratorState serialization.
        throw new IllegalDataFormatException("Unable to deserialize `serializedState`.", ex);
    }
    if (fromHash == null) {
        // Nothing to iterate on.
        return CompletableFuture.completedFuture(TableIterator.empty());
    }
    // Create a converter that will use a TableBucketReader to fetch all requested items in the iterated Buckets.
    val bucketReader = createBucketReader.apply(segment, this.keyIndex::getBackpointerOffset, this.executor);
    TableIterator.ConvertResult<IteratorItem<T>> converter = bucket -> bucketReader.findAllExisting(bucket.getSegmentOffset(), new TimeoutTimer(args.getFetchTimeout())).thenApply(result -> new IteratorItemImpl<>(new IteratorStateImpl(bucket.getHash()).serialize(), result));
    // Fetch the Tail (Unindexed) Hashes, then create the TableIterator.
    return this.keyIndex.getUnindexedKeyHashes(segment).thenComposeAsync(cacheHashes -> TableIterator.<IteratorItem<T>>builder().segment(segment).cacheHashes(cacheHashes).firstHash(fromHash).executor(executor).resultConverter(converter).fetchTimeout(args.getFetchTimeout()).build(), this.executor);
}

Example 4 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class TableSegmentLayoutTestBase method collectIteratorItems.

private <T> List<T> collectIteratorItems(AsyncIterator<IteratorItem<T>> iterator) throws Exception {
    val result = new ArrayList<T>();
    val hashes = new HashSet<BufferView>();
    iterator.forEachRemaining(item -> {
        Assert.assertTrue("Duplicate IteratorItem.getState().", hashes.add(item.getState()));
        result.addAll(item.getEntries());
    }, executorService()).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
    return result;
}

Example 5 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class TableSegmentImplTest method testIterator.

private <T> void testIterator(Function<SegmentIteratorArgs, AsyncIterator<IteratorItem<T>>> newIterator, Supplier<ByteBuf> getLastRequestFromKey, Supplier<ByteBuf> getLastRequestToKey, Function<TableSegmentEntry, T> getItemFromEntry, Consumer<List<T>> sendReply, BiPredicate<T, T> checkItemEquality) throws Exception {
    val suggestedKeyCount = 3;
    // Generate 100 Entries and split them into batches.
    val allEntries = IntStream.range(0, 100).mapToObj(i -> versionedEntry(i * 10L, Integer.toString(i * 10), 1L)).collect(Collectors.toList());
    val inputEntries = splitIteratorInputs(allEntries);
    // Do an full iteration as well.
    inputEntries.add(allEntries);
    // Check regular iteration.
    for (int i = 0; i < inputEntries.size(); i++) {
        val entryList = inputEntries.get(i);
        SegmentIteratorArgs args = SegmentIteratorArgs.builder().fromKey(entryList.get(0).getKey().getKey()).toKey(entryList.get(entryList.size() - 1).getKey().getKey()).maxItemsAtOnce(suggestedKeyCount).build();
        // We collect iterated items in this list.
        val actualItems = new ArrayList<T>();
        val itemsToReturn = entryList.iterator();
        val tableIterator = newIterator.apply(args);
        while (itemsToReturn.hasNext()) {
            val iteratorFuture = tableIterator.getNext();
            // Verify the wire command got sent as expected.
            val requestFromKey = getLastRequestFromKey.get();
            Assert.assertEquals("Unexpected fromKey sent.", args.getFromKey(), requestFromKey);
            val requestToKey = getLastRequestToKey.get();
            Assert.assertEquals("Unexpected toKey sent.", args.getToKey(), requestToKey);
            // Send a reply.
            val expectedResult = new ArrayList<T>();
            int count = suggestedKeyCount;
            while (itemsToReturn.hasNext() && count > 0) {
                val next = itemsToReturn.next();
                expectedResult.add(getItemFromEntry.apply(next));
                args = args.next(next.getKey().getKey());
                count--;
            }
            sendReply.accept(expectedResult);
            // Check the partial result.
            val iteratorResult = iteratorFuture.get(SHORT_TIMEOUT, TimeUnit.MILLISECONDS);
            AssertExtensions.assertListEquals("Unexpected partial result.", expectedResult, iteratorResult.getItems(), checkItemEquality);
            actualItems.addAll(iteratorResult.getItems());
        }
        // Then the final result.
        val expected = entryList.stream().map(getItemFromEntry).collect(Collectors.toList());
        AssertExtensions.assertListEquals("Unexpected result.", expected, actualItems, checkItemEquality);
    }
}