Examples with AsyncIterator io.pravega.common.util.AsyncIterator used on opensource projects

Example 6 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class TableSegmentImplTest method testIterator.

private <T> void testIterator(Function<SegmentIteratorArgs, AsyncIterator<IteratorItem<T>>> newIterator, Supplier<ByteBuf> getLastRequestFromKey, Supplier<ByteBuf> getLastRequestToKey, Function<TableSegmentEntry, T> getItemFromEntry, Consumer<List<T>> sendReply, BiPredicate<T, T> checkItemEquality) throws Exception {
    val suggestedKeyCount = 3;
    // Generate 100 Entries and split them into batches.
    val allEntries = IntStream.range(0, 100).mapToObj(i -> versionedEntry(i * 10L, Integer.toString(i * 10), 1L)).collect(Collectors.toList());
    val inputEntries = splitIteratorInputs(allEntries);
    // Do an full iteration as well.
    inputEntries.add(allEntries);
    // Check regular iteration.
    for (int i = 0; i < inputEntries.size(); i++) {
        val entryList = inputEntries.get(i);
        SegmentIteratorArgs args = SegmentIteratorArgs.builder().fromKey(entryList.get(0).getKey().getKey()).toKey(entryList.get(entryList.size() - 1).getKey().getKey()).maxItemsAtOnce(suggestedKeyCount).build();
        // We collect iterated items in this list.
        val actualItems = new ArrayList<T>();
        val itemsToReturn = entryList.iterator();
        val tableIterator = newIterator.apply(args);
        while (itemsToReturn.hasNext()) {
            val iteratorFuture = tableIterator.getNext();
            // Verify the wire command got sent as expected.
            val requestFromKey = getLastRequestFromKey.get();
            Assert.assertEquals("Unexpected fromKey sent.", args.getFromKey(), requestFromKey);
            val requestToKey = getLastRequestToKey.get();
            Assert.assertEquals("Unexpected toKey sent.", args.getToKey(), requestToKey);
            // Send a reply.
            val expectedResult = new ArrayList<T>();
            int count = suggestedKeyCount;
            while (itemsToReturn.hasNext() && count > 0) {
                val next = itemsToReturn.next();
                expectedResult.add(getItemFromEntry.apply(next));
                args = args.next(next.getKey().getKey());
                count--;
            }
            sendReply.accept(expectedResult);
            // Check the partial result.
            val iteratorResult = iteratorFuture.get(SHORT_TIMEOUT, TimeUnit.MILLISECONDS);
            AssertExtensions.assertListEquals("Unexpected partial result.", expectedResult, iteratorResult.getItems(), checkItemEquality);
            actualItems.addAll(iteratorResult.getItems());
        }
        // Then the final result.
        val expected = entryList.stream().map(getItemFromEntry).collect(Collectors.toList());
        AssertExtensions.assertListEquals("Unexpected result.", expected, actualItems, checkItemEquality);
    }
}

Example 7 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class HashTableSegmentLayout method entryDeltaIterator.

@Override
AsyncIterator<IteratorItem<TableEntry>> entryDeltaIterator(@NonNull DirectSegmentAccess segment, long fromPosition, Duration fetchTimeout) {
    val segmentInfo = segment.getInfo();
    ensureSegmentType(segmentInfo.getName(), segmentInfo.getType());
    Preconditions.checkArgument(fromPosition <= segmentInfo.getLength(), "fromPosition (%s) can not exceed the length (%s) of the TableSegment.", fromPosition, segmentInfo.getLength());
    logRequest("entryDeltaIterator", segment.getSegmentId(), fromPosition);
    long compactionOffset = segmentInfo.getAttributes().getOrDefault(TableAttributes.COMPACTION_OFFSET, 0L);
    // All of the most recent keys will exist beyond the compactionOffset.
    long startOffset = Math.max(fromPosition, compactionOffset);
    // We should clear if the starting position may have been truncated out due to compaction.
    boolean shouldClear = fromPosition < compactionOffset;
    // Maximum length of the TableSegment we want to read until.
    int maxBytesToRead = (int) (segmentInfo.getLength() - startOffset);
    TableEntryDeltaIterator.ConvertResult<IteratorItem<TableEntry>> converter = item -> CompletableFuture.completedFuture(new IteratorItemImpl<TableEntry>(item.getKey().serialize(), Collections.singletonList(item.getValue())));
    return TableEntryDeltaIterator.<IteratorItem<TableEntry>>builder().segment(segment).entrySerializer(serializer).executor(executor).maxBytesToRead(maxBytesToRead).startOffset(startOffset).currentBatchOffset(fromPosition).fetchTimeout(fetchTimeout).resultConverter(converter).shouldClear(shouldClear).build();
}

Example 8 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class ScopeTest method testListStreams.

@Test(timeout = 30000)
public void testListStreams() throws Exception {
    final String scope = "test";
    final String streamName1 = "test1";
    final String streamName2 = "test2";
    final String streamName3 = "test3";
    final Map<String, Integer> foundCount = new HashMap<>();
    foundCount.put(streamName1, 0);
    foundCount.put(streamName2, 0);
    foundCount.put(streamName3, 0);
    foundCount.put(NameUtils.getMarkStreamForStream(streamName1), 0);
    foundCount.put(NameUtils.getMarkStreamForStream(streamName2), 0);
    foundCount.put(NameUtils.getMarkStreamForStream(streamName3), 0);
    StreamConfiguration config = StreamConfiguration.builder().scalingPolicy(ScalingPolicy.byEventRate(10, 2, 1)).build();
    @Cleanup Controller controller = controllerWrapper.getController();
    ClientConfig clientConfig = ClientConfig.builder().controllerURI(URI.create("tcp://localhost")).build();
    @Cleanup ConnectionPool cp = new ConnectionPoolImpl(clientConfig, new SocketConnectionFactoryImpl(clientConfig));
    controllerWrapper.getControllerService().createScope(scope, 0L).get();
    controller.createStream(scope, streamName1, config).get();
    controller.createStream(scope, streamName2, config).get();
    controller.createStream(scope, streamName3, config).get();
    @Cleanup StreamManager manager = new StreamManagerImpl(controller, cp);
    Iterator<Stream> iterator = manager.listStreams(scope);
    assertTrue(iterator.hasNext());
    Stream next = iterator.next();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    assertTrue(iterator.hasNext());
    next = iterator.next();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    assertTrue(iterator.hasNext());
    next = iterator.next();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    assertTrue(iterator.hasNext());
    next = iterator.next();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    assertTrue(iterator.hasNext());
    next = iterator.next();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    assertTrue(iterator.hasNext());
    next = iterator.next();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    assertFalse(iterator.hasNext());
    assertTrue(foundCount.entrySet().stream().allMatch(x -> x.getValue() == 1));
    AsyncIterator<Stream> asyncIterator = controller.listStreams(scope);
    next = asyncIterator.getNext().join();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    next = asyncIterator.getNext().join();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    next = asyncIterator.getNext().join();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    next = asyncIterator.getNext().join();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    next = asyncIterator.getNext().join();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    next = asyncIterator.getNext().join();
    foundCount.computeIfPresent(next.getStreamName(), (x, y) -> ++y);
    next = asyncIterator.getNext().join();
    assertNull(next);
    assertTrue(foundCount.entrySet().stream().allMatch(x -> x.getValue() == 2));
}

Example 9 with AsyncIterator

use of io.pravega.common.util.AsyncIterator in project pravega by pravega.

the class ControllerImpl method listKeyValueTables.

@Override
public AsyncIterator<KeyValueTableInfo> listKeyValueTables(String scopeName) {
    Exceptions.checkNotClosed(closed.get(), this);
    long traceId = LoggerHelpers.traceEnter(log, "listKeyValueTables", scopeName);
    long requestId = requestIdGenerator.get();
    try {
        final Function<ContinuationToken, CompletableFuture<Map.Entry<ContinuationToken, Collection<KeyValueTableInfo>>>> function = token -> this.retryConfig.runAsync(() -> {
            RPCAsyncCallback<KVTablesInScopeResponse> callback = new RPCAsyncCallback<>(requestId, "listKeyValueTables", scopeName);
            ScopeInfo scopeInfo = ScopeInfo.newBuilder().setScope(scopeName).build();
            new ControllerClientTagger(client, timeoutMillis).withTag(requestId, LIST_KEY_VALUE_TABLES, scopeName).listKeyValueTables(KVTablesInScopeRequest.newBuilder().setScope(scopeInfo).setContinuationToken(token).build(), callback);
            return callback.getFuture().thenApplyAsync(x -> {
                switch(x.getStatus()) {
                    case SCOPE_NOT_FOUND:
                        log.warn(requestId, "Scope not found: {}", scopeName);
                        throw new NoSuchScopeException();
                    case FAILURE:
                        log.warn(requestId, "Internal Server Error while trying to list streams in scope: {}", scopeName);
                        throw new RuntimeException("Failure while trying to list streams");
                    case SUCCESS:
                    // compatibility reasons
                    default:
                        List<KeyValueTableInfo> kvtList = x.getKvtablesList().stream().map(y -> new KeyValueTableInfo(y.getScope(), y.getKvtName())).collect(Collectors.toList());
                        return new AbstractMap.SimpleEntry<>(x.getContinuationToken(), kvtList);
                }
            }, this.executor);
        }, this.executor);
        return new ContinuationTokenAsyncIterator<>(function, ContinuationToken.newBuilder().build());
    } finally {
        LoggerHelpers.traceLeave(log, "listKeyValueTables", traceId);
    }
}