use of io.pravega.common.util.BufferView in project pravega by pravega.
the class ContainerEventProcessorTests method testEventRejectionOnMaxOutstanding.
public static void testEventRejectionOnMaxOutstanding(ContainerEventProcessor eventProcessorService) throws Exception {
int maxItemsProcessed = 1000;
int maxOutstandingBytes = 1024 * 1024;
int truncationDataSize = 500;
ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
AtomicLong processorResults = new AtomicLong(0);
ReusableLatch latch = new ReusableLatch();
Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
Exceptions.handleInterrupted(latch::await);
processorResults.addAndGet(l.size());
return CompletableFuture.completedFuture(null);
};
@Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegmentMax", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
boolean foundMaxOutstandingLimit = false;
// Write a lot of data with the ContainerEventProcessor not started, so we ensure we get the max outstanding exception.
BufferView event = new ByteArraySegment("This needs to be a long string to reach the limit sooner!!!".getBytes());
while (!foundMaxOutstandingLimit) {
try {
processor.add(event, TIMEOUT_FUTURE).join();
processorResults.decrementAndGet();
} catch (Exception e) {
// We have reached the max outstanding bytes for this internal Segment.
Assert.assertTrue(e instanceof ContainerEventProcessor.TooManyOutstandingBytesException);
foundMaxOutstandingLimit = true;
}
}
latch.release();
// Wait until all the events are consumed.
AssertExtensions.assertEventuallyEquals(true, () -> processorResults.get() == 0, TIMEOUT_SUITE_MILLIS);
// Check that we cannot add empty events.
AssertExtensions.assertThrows(IllegalArgumentException.class, () -> processor.add(BufferView.empty(), TIMEOUT_FUTURE));
}
use of io.pravega.common.util.BufferView in project pravega by pravega.
the class ContainerEventProcessorTests method testFaultyHandler.
public static void testFaultyHandler(ContainerEventProcessor eventProcessorService) throws Exception {
int maxItemsProcessed = 10;
int maxOutstandingBytes = 4 * 1024 * 1024;
int truncationDataSize = 500;
AtomicLong retries = new AtomicLong(0);
Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
retries.addAndGet(1);
// Induce some exception here.
throw new IntentionalException("Some random failure");
};
ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
@Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSegment", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
// Write an event and wait for the event to be processed.
BufferView event = new ByteArraySegment("event".getBytes());
processor.add(event, TIMEOUT_FUTURE).join();
// Wait until the processor perform 10 retries of the same event.
AssertExtensions.assertEventuallyEquals(true, () -> retries.get() == 10, 10000);
}
use of io.pravega.common.util.BufferView in project pravega by pravega.
the class ContainerEventProcessorTests method testTruncationSizeRespected.
/**
* Verifies that we truncate the Segment the expected number of times based on the config.
*
* @throws Exception
*/
@Test(timeout = TIMEOUT_SUITE_MILLIS)
public void testTruncationSizeRespected() throws Exception {
MockSegmentSupplier mockSegmentSupplier = new MockSegmentSupplier();
@Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, mockSegmentSupplier.mockSegmentSupplier(), ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
int allEventsToProcess = 10;
int maxItemsPerBatch = 10;
int maxOutstandingBytes = 4 * 1024 * 1024;
int truncationDataSize = 20;
List<Integer> processorResults1 = new ArrayList<>();
Function<List<BufferView>, CompletableFuture<Void>> handler1 = getNumberSequenceHandler(processorResults1, maxItemsPerBatch);
ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize);
@Cleanup ContainerEventProcessorImpl.EventProcessorImpl processor1 = (ContainerEventProcessorImpl.EventProcessorImpl) eventProcessorService.forConsumer("testSegment1", handler1, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
for (int i = 0; i < allEventsToProcess; i++) {
BufferView event = new ByteArraySegment(ByteBuffer.allocate(Integer.BYTES).putInt(i).array());
processor1.add(event, TIMEOUT_FUTURE).join();
// After adding one event, wait for it to be processed.
AssertExtensions.assertEventuallyEquals(true, () -> processor1.getOutstandingBytes() == 0, 10000);
}
// Wait for all items to be processed.
validateProcessorResults(processor1, processorResults1, allEventsToProcess);
// Each Integer event write is 13 bytes in size after being serialized.
verify(mockSegmentSupplier.segmentMock, times((13 * allEventsToProcess) / truncationDataSize - 1)).truncate(anyLong(), any());
}
use of io.pravega.common.util.BufferView in project pravega by pravega.
the class ContainerEventProcessorTests method executeEventOrderingTest.
private void executeEventOrderingTest(int numEvents, List<Integer> readEvents, Consumer<DirectSegmentAccess> segmentFailure) throws Exception {
DirectSegmentAccess faultySegment = spy(new SegmentMock(this.executorService()));
Function<String, CompletableFuture<DirectSegmentAccess>> faultySegmentSupplier = s -> CompletableFuture.completedFuture(faultySegment);
@Cleanup ContainerEventProcessor eventProcessorService = new ContainerEventProcessorImpl(0, faultySegmentSupplier, ITERATION_DELAY, CONTAINER_OPERATION_TIMEOUT, this.executorService());
int maxItemsProcessed = 100;
int maxOutstandingBytes = 4 * 1024 * 1024;
int truncationDataSize = 500;
Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
l.forEach(d -> readEvents.add(new ByteArraySegment(d.getCopy()).getInt(0)));
return CompletableFuture.completedFuture(null);
};
// Induce some desired failures.
segmentFailure.accept(faultySegment);
ContainerEventProcessor.EventProcessorConfig config = new ContainerEventProcessor.EventProcessorConfig(maxItemsProcessed, maxOutstandingBytes, truncationDataSize);
@Cleanup ContainerEventProcessor.EventProcessor processor = eventProcessorService.forConsumer("testSequence", handler, config).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
// Write some data.
for (int i = 0; i < numEvents; i++) {
ByteArraySegment b = new ByteArraySegment(new byte[Integer.BYTES]);
b.setInt(0, i);
processor.add(b.slice(), TIMEOUT_FUTURE).join();
}
// Wait until the last event is read.
AssertExtensions.assertEventuallyEquals(true, () -> !readEvents.isEmpty() && readEvents.get(readEvents.size() - 1) == numEvents - 1, 10000);
}
use of io.pravega.common.util.BufferView in project pravega by pravega.
the class ContainerEventProcessorTests method testBasicContainerEventProcessor.
public static void testBasicContainerEventProcessor(ContainerEventProcessor containerEventProcessor) throws Exception {
int maxItemsPerBatch = 10;
int maxOutstandingBytes = 4 * 1024 * 1024;
int truncationDataSize = 500;
ReusableLatch latch = new ReusableLatch();
final AtomicReference<String> userEvent = new AtomicReference<>("event1");
Function<List<BufferView>, CompletableFuture<Void>> handler = l -> {
l.forEach(s -> Assert.assertEquals(userEvent.get().length(), s.getLength()));
return CompletableFuture.runAsync(latch::release);
};
ContainerEventProcessor.EventProcessorConfig eventProcessorConfig = new ContainerEventProcessor.EventProcessorConfig(maxItemsPerBatch, maxOutstandingBytes, truncationDataSize);
@Cleanup ContainerEventProcessor.EventProcessor processor = containerEventProcessor.forConsumer("testConsumer", handler, eventProcessorConfig).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS);
// Test adding one Event to the EventProcessor and wait for the handle to get executed and unblock this thread.
BufferView event = new ByteArraySegment(userEvent.get().getBytes());
processor.add(event, TIMEOUT_FUTURE).join();
latch.await();
latch.reset();
// Test the same with a larger events.
userEvent.set("longerEvent2");
event = new ByteArraySegment(userEvent.get().getBytes());
processor.add(event, TIMEOUT_FUTURE).join();
processor.add(event, TIMEOUT_FUTURE).join();
processor.add(event, TIMEOUT_FUTURE).join();
latch.await();
// Check that if the same EventProcessor name is used, the same object is retrieved.
Assert.assertEquals(processor, containerEventProcessor.forConsumer("testConsumer", handler, eventProcessorConfig).get(TIMEOUT_FUTURE.toSeconds(), TimeUnit.SECONDS));
}
Aggregations