use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.
the class EventProcessorTest method testEventProcessorCell.
@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testEventProcessorCell() throws CheckpointStoreException, ReinitializationRequiredException {
CheckpointStore checkpointStore = CheckpointStoreFactory.createInMemoryStore();
CheckpointConfig.CheckpointPeriod period = CheckpointConfig.CheckpointPeriod.builder().numEvents(1).numSeconds(1).build();
CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.Periodic).checkpointPeriod(period).build();
EventProcessorGroupConfig config = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(READER_GROUP).streamName(STREAM_NAME).checkpointConfig(checkpointConfig).build();
int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
int expectedSum = input.length * (input.length + 1) / 2;
List<MockEventRead<TestEvent>> inputEvents = new ArrayList<>(input.length);
for (int i = 0; i < input.length; i++) {
inputEvents.add(new MockEventRead<>(i, new TestEvent(input[i])));
}
inputEvents.add(new MockEventRead<>(input.length, new TestEvent(-1)));
EventProcessorSystem system = Mockito.mock(EventProcessorSystem.class);
Mockito.when(system.getProcess()).thenReturn(PROCESS);
EventStreamReader<TestEvent> reader = Mockito.mock(EventStreamReader.class);
checkpointStore.addReaderGroup(PROCESS, READER_GROUP);
// Test case 1. Actor does not throw any exception during normal operation.
Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, expectedSum);
// Test case 2. Actor throws an error during normal operation, and Directive is to Resume on error.
Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(true)).serializer(new JavaSerializer<>()).decider((Throwable e) -> (e instanceof IllegalArgumentException) ? ExceptionHandler.Directive.Resume : ExceptionHandler.Directive.Stop).config(config).build();
testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, expectedSum);
// Test case 3. Actor throws an error during normal operation, and Directive is to Restart on error.
Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(true)).serializer(new JavaSerializer<>()).decider((Throwable e) -> (e instanceof IllegalArgumentException) ? ExceptionHandler.Directive.Restart : ExceptionHandler.Directive.Stop).config(config).build();
testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, 0);
// Test case 3. Actor throws an error during normal operation, and Directive is to Restart on error.
Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new RestartFailingEventProcessor(true)).serializer(new JavaSerializer<>()).decider((Throwable e) -> (e instanceof IllegalArgumentException) ? ExceptionHandler.Directive.Restart : ExceptionHandler.Directive.Stop).config(config).build();
testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, 3);
// Test case 5. startup fails for an event processor
eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(StartFailingEventProcessor::new).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
checkpointStore.addReader(PROCESS, READER_GROUP, READER_ID);
EventProcessorCell<TestEvent> cell = new EventProcessorCell<>(eventProcessorConfig, reader, new EventStreamWriterMock<>(), system.getProcess(), READER_ID, 0, checkpointStore);
cell.startAsync();
cell.awaitTerminated();
Assert.assertTrue(true);
}
use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.
the class EventProcessorTest method testEventProcessorGroup.
@Test(timeout = 10000)
public void testEventProcessorGroup() throws CheckpointStoreException, ReinitializationRequiredException {
int count = 4;
int initialCount = count / 2;
String systemName = "testSystem";
String readerGroupName = "testReaderGroup";
int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
int expectedSum = input.length * (input.length + 1) / 2;
CheckpointStore checkpointStore = CheckpointStoreFactory.createInMemoryStore();
EventProcessorGroupConfig config = createEventProcessorGroupConfig(initialCount);
EventProcessorSystemImpl system = createMockSystem(systemName, PROCESS, SCOPE, createEventReaders(count, input), new EventStreamWriterMock<>(), readerGroupName);
EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
// Create EventProcessorGroup.
EventProcessorGroupImpl<TestEvent> group = (EventProcessorGroupImpl<TestEvent>) system.createEventProcessorGroup(eventProcessorConfig, checkpointStore);
group.awaitRunning();
// Add a few event processors to the group.
group.changeEventProcessorCount(count - initialCount);
long actualSum = 0;
for (EventProcessorCell<TestEvent> cell : group.getEventProcessorMap().values()) {
cell.awaitTerminated();
TestEventProcessor actor = (TestEventProcessor) cell.getActor();
actualSum += actor.sum;
}
assertEquals(count * expectedSum, actualSum);
// Stop the group, and await its termmination.
group.stopAsync();
group.awaitTerminated();
}
use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.
the class EventProcessorTest method testEventProcessor.
@Test(timeout = 60000)
public void testEventProcessor() throws Exception {
@Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
serviceBuilder.initialize();
StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
int servicePort = TestUtils.getAvailableListenPort();
@Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
server.startListening();
int controllerPort = TestUtils.getAvailableListenPort();
@Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), true, controllerPort, "localhost", servicePort, 4);
controllerWrapper.awaitRunning();
Controller controller = controllerWrapper.getController();
// Create controller object for testing against a separate controller process.
// ControllerImpl controller = new ControllerImpl("localhost", 9090);
final String host = "host";
final String scope = "controllerScope";
final String streamName = "stream1";
final String readerGroup = "readerGroup";
final CompletableFuture<Boolean> createScopeStatus = controller.createScope(scope);
if (!createScopeStatus.join()) {
throw new RuntimeException("Scope already existed");
}
final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
System.err.println(String.format("Creating stream (%s, %s)", scope, streamName));
CompletableFuture<Boolean> createStatus = controller.createStream(config);
if (!createStatus.get()) {
System.err.println("Stream alrady existed, exiting");
return;
}
@Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
@Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
@Cleanup EventStreamWriter<TestEvent> producer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
int expectedSum = input.length * (input.length + 1) / 2;
for (int i = 0; i < input.length; i++) {
producer.writeEvent("key", new TestEvent(input[i]));
}
producer.writeEvent("key", new TestEvent(-1));
producer.flush();
EventProcessorSystem system = new EventProcessorSystemImpl("Controller", host, scope, new ClientFactoryImpl(scope, controller, connectionFactory), new ReaderGroupManagerImpl(scope, controller, clientFactory, connectionFactory));
CheckpointConfig.CheckpointPeriod period = CheckpointConfig.CheckpointPeriod.builder().numEvents(1).numSeconds(1).build();
CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.Periodic).checkpointPeriod(period).build();
EventProcessorGroupConfig eventProcessorGroupConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroup).streamName(streamName).checkpointConfig(checkpointConfig).build();
CompletableFuture<Long> result = new CompletableFuture<>();
// Test case 1. Actor does not throw any exception during normal operation.
EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false, result)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(eventProcessorGroupConfig).build();
@Cleanup EventProcessorGroup<TestEvent> eventProcessorGroup = system.createEventProcessorGroup(eventProcessorConfig, CheckpointStoreFactory.createInMemoryStore());
Long value = result.join();
Assert.assertEquals(expectedSum, value.longValue());
log.info("SUCCESS: received expected sum = " + expectedSum);
}
use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.
the class StreamTransactionMetadataTasksTest method createEventProcessor.
private <T extends ControllerEvent> void createEventProcessor(final String readerGroupName, final String streamName, final EventStreamReader<T> reader, final EventStreamWriter<T> writer, Supplier<EventProcessor<T>> factory) throws CheckpointStoreException {
ClientFactory clientFactory = Mockito.mock(ClientFactory.class);
Mockito.when(clientFactory.<T>createReader(anyString(), anyString(), any(), any())).thenReturn(reader);
Mockito.when(clientFactory.<T>createEventWriter(anyString(), any(), any())).thenReturn(writer);
ReaderGroup readerGroup = Mockito.mock(ReaderGroup.class);
Mockito.when(readerGroup.getGroupName()).thenReturn(readerGroupName);
ReaderGroupManager readerGroupManager = Mockito.mock(ReaderGroupManager.class);
Mockito.when(readerGroupManager.createReaderGroup(anyString(), any(ReaderGroupConfig.class))).then(invocation -> readerGroup);
EventProcessorSystemImpl system = new EventProcessorSystemImpl("system", "host", SCOPE, clientFactory, readerGroupManager);
EventProcessorGroupConfig eventProcessorConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroupName).streamName(streamName).checkpointConfig(CheckpointConfig.periodic(1, 1)).build();
EventProcessorConfig<T> config = EventProcessorConfig.<T>builder().config(eventProcessorConfig).decider(ExceptionHandler.DEFAULT_EXCEPTION_HANDLER).serializer(new JavaSerializer<>()).supplier(factory).build();
system.createEventProcessorGroup(config, CheckpointStoreFactory.createInMemoryStore());
}
use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.
the class EventProcessorTest method testFailingEventProcessorInGroup.
@Test(timeout = 10000)
public void testFailingEventProcessorInGroup() throws ReinitializationRequiredException, CheckpointStoreException {
String systemName = "testSystem";
String readerGroupName = "testReaderGroup";
int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
CheckpointStore checkpointStore = CheckpointStoreFactory.createInMemoryStore();
EventProcessorGroupConfig config = createEventProcessorGroupConfig(1);
EventProcessorSystemImpl system = createMockSystem(systemName, PROCESS, SCOPE, createEventReaders(1, input), new EventStreamWriterMock<>(), readerGroupName);
EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(StartFailingEventProcessor::new).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
// Create EventProcessorGroup.
EventProcessorGroupImpl<TestEvent> group = (EventProcessorGroupImpl<TestEvent>) system.createEventProcessorGroup(eventProcessorConfig, checkpointStore);
// awaitRunning should succeed.
group.awaitRunning();
Assert.assertTrue(true);
}
Aggregations