Examples with ContainerEventProcessor io.pravega.segmentstore.server.ContainerEventProcessor used on opensource projects

Example 21 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testTruncationSizeRespected.

/**
 * Verifies that we truncate the Segment the expected number of times based on the config.
 *
 * @throws Exception
 */
@Test(timeout = TIMEOUT_SUITE_MILLIS)
public void testTruncationSizeRespected() throws Exception {
    MockSegmentSupplier mockSegmentSupplier = new MockSegmentSupplier();
    @Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, mockSegmentSupplier.mockSegmentSupplier(), ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
    int allEventsToProcess = 10;
    int maxItemsPerBatch = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 20;
    List<Integer> processorResults1 = new ArrayList<>();
    Function<List<BufferView>, CompletableFuture<Void>> handler1 = getNumberSequenceHandler(processorResults1, maxItemsPerBatch);
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize);
    @Cleanup ContainerEventProcessorImpl.EventProcessorImpl processor1 = (ContainerEventProcessorImpl.EventProcessorImpl) eventProcessorService.forConsumer("testSegment1", handler1, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    for (int i = 0; i < allEventsToProcess; i++) {
        BufferView event = new ByteArraySegment(ByteBuffer.allocate(Integer.BYTES).putInt(i).array());
        processor1.add(event, TIMEOUT_FUTURE).join();
        // After adding one event, wait for it to be processed.
        AssertExtensions.assertEventuallyEquals(true, () -> processor1.getOutstandingBytes() == 0, 10000);
    }
    // Wait for all items to be processed.
    validateProcessorResults(processor1, processorResults1, allEventsToProcess);
    // Each Integer event write is 13 bytes in size after being serialized.
    verify(mockSegmentSupplier.segmentMock, times((13 * allEventsToProcess) / truncationDataSize - 1)).truncate(anyLong(), any());
}

Example 22 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method executeEventOrderingTest.

private void executeEventOrderingTest(int numEvents, List<Integer> readEvents, Consumer<DirectSegmentAccess> segmentFailure) throws Exception {
    DirectSegmentAccess faultySegment = spy(new SegmentMock(this.executorService()));
    Function<String, CompletableFuture<DirectSegmentAccess>> faultySegmentSupplier = s -> CompletableFuture.completedFuture(faultySegment);
    @Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, faultySegmentSupplier, ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
    int maxItemsProcessed = 100;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
        l.forEach(d -> readEvents.add(new ByteArraySegment(d.getCopy()).getInt(0)));
        return CompletableFuture.completedFuture(null);
    };
    // Induce some desired failures.
    segmentFailure.accept(faultySegment);
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSequence", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Write some data.
    for (int i = 0; i < numEvents; i++) {
        ByteArraySegment b = new ByteArraySegment(new byte[Integer.BYTES]);
        b.setInt(0, i);
        processor.add(b.slice(), TIMEOUT_FUTURE).join();
    }
    // Wait until the last event is read.
    AssertExtensions.assertEventuallyEquals(true, () -> !readEvents.isEmpty() && readEvents.get(readEvents.size() - 1) == numEvents - 1, 10000);
}

Example 23 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testBasicContainerEventProcessor.

public static void testBasicContainerEventProcessor(ContainerEventProcessor containerEventProcessor) throws Exception {
    int maxItemsPerBatch = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    ReusableLatch latch = new ReusableLatch();
    final AtomicReference<String> userEvent = new AtomicReference<>("event1");
    Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
        l.forEach(s -> Assert.assertEquals(userEvent.get().length(), s.getLength()));
        return CompletableFuture.runAsync(latch::release);
    };
    ContainerEventProcessor.EventProcessorConfig eventProcessorConfig = new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize);
    @Cleanup ContainerEventProcessor.EventProcessor processor = containerEventProcessor.forConsumer("testConsumer", handler, eventProcessorConfig).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Test adding one Event to the EventProcessor and wait for the handle to get executed and unblock this thread.
    BufferView event = new ByteArraySegment(userEvent.get().getBytes());
    processor.add(event, TIMEOUT_FUTURE).join();
    latch.await();
    latch.reset();
    // Test the same with a larger events.
    userEvent.set("longerEvent2");
    event = new ByteArraySegment(userEvent.get().getBytes());
    processor.add(event, TIMEOUT_FUTURE).join();
    processor.add(event, TIMEOUT_FUTURE).join();
    processor.add(event, TIMEOUT_FUTURE).join();
    latch.await();
    // Check that if the same EventProcessor name is used, the same object is retrieved.
    Assert.assertEquals(processor, containerEventProcessor.forConsumer("testConsumer", handler, eventProcessorConfig).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS));
}

Example 24 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testEventProcessorWithSerializationError.

/**
 * Test the situation in which an EventProcessor gets BufferView.Reader.OutOfBoundsException during deserialization
 * of events.
 *
 * @throws Exception
 */
@Test(timeout = 10000)
public void testEventProcessorWithSerializationError() throws Exception {
    @Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, mockSegmentSupplier(), ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
    testEventProcessorWithSerializationError(eventProcessorService);
}