Examples with BufferView io.pravega.common.util.BufferView used on opensource projects

Example 41 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class ContainerEventProcessorTests method testEventProcessorClose.

/**
 * Test closing the EventProcessor.
 *
 * @throws Exception
 */
@Test(timeout = 30000)
public void testEventProcessorClose() throws Exception {
    @Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, mockSegmentSupplier(), ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
    int maxItemsProcessed = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
    AtomicLong processorResults = new AtomicLong(0);
    Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
        processorResults.addAndGet(l.size());
        return CompletableFuture.completedFuture(null);
    };
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testClose", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Assert that the existing object in the ContainerEventProcessorImpl map is the same as the one we just instantiated.
    Assert.assertEquals(processor, eventProcessorService.forConsumer("testClose", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS));
    // Now, close the EventProcessor object, which should auto-unregister from the map.
    processor.close();
    // After that, we should see a new object being instantiated in ContainerEventProcessorImpl for the same name.
    Assert.assertNotEquals(processor, eventProcessorService.forConsumer("testClose", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS));
}

Example 42 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class ContainerEventProcessorTests method testEventProcessorWithSerializationError.

public static void testEventProcessorWithSerializationError(ContainerEventProcessor containerEventProcessor) throws Exception {
    int maxItemsPerBatch = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    AtomicLong readEvents = new AtomicLong(0);
    Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
        readEvents.addAndGet(l.size());
        return CompletableFuture.completedFuture(null);
    };
    ContainerEventProcessor.EventProcessorConfig eventProcessorConfig = spy(new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize));
    @Cleanup ContainerEventProcessor.EventProcessor processor = containerEventProcessor.forConsumer("testConsumer", handler, eventProcessorConfig).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Simulate an BufferView.Reader.OutOfBoundsException within the deserializeEvents() method and then behave normally.
    when(eventProcessorConfig.getMaxItemsAtOnce()).thenThrow(new BufferView.Reader.OutOfBoundsException()).thenThrow(new RuntimeException(new SerializationException("Intentional exception"))).thenCallRealMethod();
    // Write an event and wait for the event to be processed.
    BufferView event = new ByteArraySegment("event".getBytes());
    processor.add(event, TIMEOUT_FUTURE).join();
    // Wait until the processor reads the event.
    AssertExtensions.assertEventuallyEquals(true, () -> readEvents.get() == 1, 10000);
}

Example 43 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class ContainerEventProcessorTests method testMultipleProcessors.

public static void testMultipleProcessors(ContainerEventProcessor eventProcessorService) throws Exception {
    int allEventsToProcess = 100;
    int maxItemsPerBatch = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    List<Integer> processorResults1 = new ArrayList<>();
    Function<List<BufferView>, CompletableFuture<Void>> handler1 = getNumberSequenceHandler(processorResults1, maxItemsPerBatch);
    List<Integer> processorResults2 = new ArrayList<>();
    Function<List<BufferView>, CompletableFuture<Void>> handler2 = getNumberSequenceHandler(processorResults2, maxItemsPerBatch);
    AtomicLong processorResults3 = new AtomicLong(0);
    Function<List<BufferView>, CompletableFuture<Void>> handler3 = l -> {
        Object o = null;
        // We should expect NPE here, so the results counter would not be incremented.
        o.toString();
        processorResults3.addAndGet(1);
        return CompletableFuture.completedFuture(null);
    };
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize);
    @Cleanup ContainerEventProcessor.EventProcessor processor1 = eventProcessorService.forConsumer("testSegment1", handler1, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    @Cleanup ContainerEventProcessor.EventProcessor processor2 = eventProcessorService.forConsumer("testSegment2", handler2, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    @Cleanup ContainerEventProcessor.EventProcessor processor3 = eventProcessorService.forConsumer("testSegment3", handler3, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    for (int i = 0; i < allEventsToProcess; i++) {
        BufferView event = new ByteArraySegment(ByteBuffer.allocate(Integer.BYTES).putInt(i).array());
        processor1.add(event, TIMEOUT_FUTURE).join();
        processor2.add(event, TIMEOUT_FUTURE).join();
        processor3.add(event, TIMEOUT_FUTURE).join();
    }
    // Wait for all items to be processed.
    validateProcessorResults(processor1, processorResults1, allEventsToProcess);
    validateProcessorResults(processor2, processorResults2, allEventsToProcess);
    AssertExtensions.assertEventuallyEquals(true, () -> processorResults3.get() == 0, 10000);
    AssertExtensions.assertEventuallyEquals(true, () -> ((ContainerEventProcessorImpl.EventProcessorImpl) processor3).getOutstandingBytes() > 0, 10000);
}

Example 44 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class PravegaRequestProcessor method handleReadResult.

/**
 * Handles a readResult.
 * If there are cached entries that can be returned without blocking only these are returned.
 * Otherwise the call will request the data and setup a callback to return the data when it is available.
 * If no data is available but it was detected that the Segment had been truncated beyond the current offset,
 * an appropriate message is sent back over the connection.
 */
private void handleReadResult(ReadSegment request, ReadResult result) {
    String segment = request.getSegment();
    ArrayList<BufferView> cachedEntries = new ArrayList<>();
    ReadResultEntry nonCachedEntry = collectCachedEntries(request.getOffset(), result, cachedEntries);
    final String operation = "readSegment";
    boolean truncated = nonCachedEntry != null && nonCachedEntry.getType() == Truncated;
    boolean endOfSegment = nonCachedEntry != null && nonCachedEntry.getType() == EndOfStreamSegment;
    boolean atTail = nonCachedEntry != null && nonCachedEntry.getType() == Future;
    if (!cachedEntries.isEmpty() || endOfSegment) {
        // We managed to collect some data. Send it.
        ByteBuf data = toByteBuf(cachedEntries);
        SegmentRead reply = new SegmentRead(segment, request.getOffset(), atTail, endOfSegment, data, request.getRequestId());
        connection.send(reply);
        this.statsRecorder.read(segment, reply.getData().readableBytes());
    } else if (truncated) {
        // We didn't collect any data, instead we determined that the current read offset was truncated.
        // Determine the current Start Offset and send that back.
        segmentStore.getStreamSegmentInfo(segment, TIMEOUT).thenAccept(info -> connection.send(new SegmentIsTruncated(request.getRequestId(), segment, info.getStartOffset(), EMPTY_STACK_TRACE, nonCachedEntry.getStreamSegmentOffset()))).exceptionally(e -> handleException(request.getRequestId(), segment, nonCachedEntry.getStreamSegmentOffset(), operation, wrapCancellationException(e)));
    } else {
        Preconditions.checkState(nonCachedEntry != null, "No ReadResultEntries returned from read!?");
        nonCachedEntry.requestContent(TIMEOUT);
        nonCachedEntry.getContent().thenAccept(contents -> {
            ByteBuf data = toByteBuf(Collections.singletonList(contents));
            SegmentRead reply = new SegmentRead(segment, nonCachedEntry.getStreamSegmentOffset(), atTail, endOfSegment, data, request.getRequestId());
            connection.send(reply);
            this.statsRecorder.read(segment, reply.getData().readableBytes());
        }).exceptionally(exception -> {
            Throwable e = Exceptions.unwrap(exception);
            if (e instanceof StreamSegmentTruncatedException) {
                // The Segment may have been truncated in Storage after we got this entry but before we managed
                // to make a read. In that case, send the appropriate error back.
                final String clientReplyStackTrace = replyWithStackTraceOnError ? e.getMessage() : EMPTY_STACK_TRACE;
                connection.send(new SegmentIsTruncated(request.getRequestId(), segment, ((StreamSegmentTruncatedException) e).getStartOffset(), clientReplyStackTrace, nonCachedEntry.getStreamSegmentOffset()));
            } else {
                handleException(request.getRequestId(), segment, nonCachedEntry.getStreamSegmentOffset(), operation, wrapCancellationException(e));
            }
            return null;
        }).exceptionally(e -> handleException(request.getRequestId(), segment, nonCachedEntry.getStreamSegmentOffset(), operation, wrapCancellationException(e)));
    }
}

Example 45 with BufferView

use of io.pravega.common.util.BufferView in project pravega by pravega.

the class PravegaRequestProcessor method readTable.

@Override
public void readTable(final WireCommands.ReadTable readTable) {
    final String segment = readTable.getSegment();
    final String operation = "readTable";
    if (!verifyToken(segment, readTable.getRequestId(), readTable.getDelegationToken(), operation)) {
        readTable.release();
        return;
    }
    log.debug(readTable.getRequestId(), "Get Table Segment Keys: Segment={}, Count={}.", readTable.getSegment(), readTable.getKeys());
    final List<BufferView> keys = readTable.getKeys().stream().map(k -> new ByteBufWrapper(k.getData())).collect(Collectors.toList());
    val timer = new Timer();
    tableStore.get(segment, keys, TIMEOUT).thenAccept(values -> {
        connection.send(new WireCommands.TableRead(readTable.getRequestId(), segment, getTableEntriesCommand(keys, values)));
        this.tableStatsRecorder.getKeys(readTable.getSegment(), keys.size(), timer.getElapsed());
    }).exceptionally(e -> handleException(readTable.getRequestId(), segment, operation, e)).whenComplete((r, ex) -> readTable.release());
}