Examples with ContainerEventProcessor io.pravega.segmentstore.server.ContainerEventProcessor used on opensource projects

Example 16 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testReadWithFailingSegment.

/**
 * Test the behavior of the EventProcessor when internal Segment reads fail.
 *
 * @throws Exception
 */
@Test(timeout = 10000)
public void testReadWithFailingSegment() throws Exception {
    DirectSegmentAccess faultySegment = spy(new SegmentMock(this.executorService()));
    Function<String, CompletableFuture<DirectSegmentAccess>> faultySegmentSupplier = s -> CompletableFuture.completedFuture(faultySegment);
    @Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, faultySegmentSupplier, ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
    int maxItemsProcessed = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
    ReusableLatch latch = new ReusableLatch();
    Function<List<BufferView>, CompletableFuture<Void>> doNothing = l -> {
        latch.release();
        return CompletableFuture.completedFuture(null);
    };
    // Make the internal Segment of the processor to fail upon a read.
    when(faultySegment.read(anyLong(), anyInt(), any(Duration.class))).thenThrow(IntentionalException.class).thenCallRealMethod();
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegmentMax", doNothing, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Write an event to make sure that the processor is running and await for it to be processed.
    BufferView event = new ByteArraySegment("Test".getBytes());
    processor.add(event, TIMEOUT_FUTURE).join();
    latch.await();
}

Example 17 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testConcurrentForConsumerCall.

/**
 * Test the behavior of the EventProcessor when calling concurrently to forConsumer() and forDurableQueue() methods.
 *
 * @throws Exception
 */
@Test(timeout = 30000)
public void testConcurrentForConsumerCall() throws Exception {
    @Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, mockSegmentSupplier(), ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
    int maxItemsProcessed = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
    // Try to instantiate the same EventProcessor concurrently.
    CompletableFuture<ContainerEventProcessor.EventProcessor> ep1 = eventProcessorService.forConsumer("testConcurrentForConsumer", l -> null, config);
    CompletableFuture<ContainerEventProcessor.EventProcessor> ep2 = eventProcessorService.forConsumer("testConcurrentForConsumer", l -> null, config);
    CompletableFuture<ContainerEventProcessor.EventProcessor> ep3 = eventProcessorService.forConsumer("testConcurrentForConsumer", l -> null, config);
    // Wait for all these calls to complete.
    CompletableFuture.allOf(ep1, ep2, ep3).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // When instantiating an EventProcessor, the service should be started.
    Assert.assertTrue(((ContainerEventProcessorImpl.EventProcessorImpl) ep1.join()).isRunning());
    // Ensure that all EventProcessors are the same object.
    Assert.assertTrue(ep1.join() == ep2.join());
    Assert.assertTrue(ep2.join() == ep3.join());
    // Try to instantiate the same EventProcessor concurrently.
    ep1 = eventProcessorService.forDurableQueue("testConcurrentForDurableQueue");
    ep2 = eventProcessorService.forDurableQueue("testConcurrentForDurableQueue");
    ep3 = eventProcessorService.forDurableQueue("testConcurrentForDurableQueue");
    // When instantiating an EventProcessor as a durable queue, the service should not be started (we just do adds).
    Assert.assertFalse(((ContainerEventProcessorImpl.EventProcessorImpl) ep1.join()).isRunning());
    // Wait for all these calls to complete.
    CompletableFuture.allOf(ep1, ep2, ep3).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Ensure that all EventProcessors are the same object.
    Assert.assertTrue(ep1.join() == ep2.join());
    Assert.assertTrue(ep2.join() == ep3.join());
}

Example 18 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testContainerMaxItemsPerBatchRespected.

public static void testContainerMaxItemsPerBatchRespected(ContainerEventProcessor eventProcessorService) throws Exception {
    int maxItemsPerBatch = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int allEventsToProcess = 1000;
    int truncationDataSize = 500;
    List<Integer> processedItems = new ArrayList<>();
    Function<List<BufferView>, CompletableFuture<Void>> handler = getNumberSequenceHandler(processedItems, maxItemsPerBatch);
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize);
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegment", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Write all the events as fast as possible.
    for (int i = 0; i < allEventsToProcess; i++) {
        BufferView event = new ByteArraySegment(ByteBuffer.allocate(Integer.BYTES).putInt(i).array());
        processor.add(event, TIMEOUT_FUTURE).join();
    }
    // Perform basic validation on this processor.
    validateProcessorResults(processor, processedItems, allEventsToProcess);
}

Example 19 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testEventRejectionOnMaxOutstanding.

public static void testEventRejectionOnMaxOutstanding(ContainerEventProcessor eventProcessorService) throws Exception {
    int maxItemsProcessed = 1000;
    int maxOutstandingBytes = 1024 * 1024;
    int truncationDataSize = 500;
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
    AtomicLong processorResults = new AtomicLong(0);
    ReusableLatch latch = new ReusableLatch();
    Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
        Exceptions.handleInterrupted(latch::await);
        processorResults.addAndGet(l.size());
        return CompletableFuture.completedFuture(null);
    };
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegmentMax", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    boolean foundMaxOutstandingLimit = false;
    // Write a lot of data with the ContainerEventProcessor not started, so we ensure we get the max outstanding exception.
    BufferView event = new ByteArraySegment("This needs to be a long string to reach the limit sooner!!!".getBytes());
    while (!foundMaxOutstandingLimit) {
        try {
            processor.add(event, TIMEOUT_FUTURE).join();
            processorResults.decrementAndGet();
        } catch (Exception e) {
            // We have reached the max outstanding bytes for this internal Segment.
            Assert.assertTrue(e instanceof ContainerEventProcessor.TooManyOutstandingBytesException);
            foundMaxOutstandingLimit = true;
        }
    }
    latch.release();
    // Wait until all the events are consumed.
    AssertExtensions.assertEventuallyEquals(true, () -> processorResults.get() == 0, TIMEOUT_SUITE_MILLIS);
    // Check that we cannot add empty events.
    AssertExtensions.assertThrows(IllegalArgumentException.class, () -> processor.add(BufferView.empty(), TIMEOUT_FUTURE));
}

Example 20 with ContainerEventProcessor

use of io.pravega.segmentstore.server.ContainerEventProcessor in project pravega by pravega.

the class ContainerEventProcessorTests method testFaultyHandler.

public static void testFaultyHandler(ContainerEventProcessor eventProcessorService) throws Exception {
    int maxItemsProcessed = 10;
    int maxOutstandingBytes = 4 * 1024 * 1024;
    int truncationDataSize = 500;
    AtomicLong retries = new AtomicLong(0);
    Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
        retries.addAndGet(1);
        // Induce some exception here.
        throw new IntentionalException("Some random failure");
    };
    ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
    @Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegment", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
    // Write an event and wait for the event to be processed.
    BufferView event = new ByteArraySegment("event".getBytes());
    processor.add(event, TIMEOUT_FUTURE).join();
    // Wait until the processor perform 10 retries of the same event.
    AssertExtensions.assertEventuallyEquals(true, () -> retries.get() == 10, 10000);
}