Search in sources :

Example 6 with EventProcessorGroupConfig

use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.

the class EventProcessorTest method testEventProcessorWriter.

@Test(timeout = 10000)
public void testEventProcessorWriter() throws ReinitializationRequiredException, CheckpointStoreException {
    int initialCount = 1;
    String systemName = "testSystem";
    String readerGroupName = "testReaderGroup";
    int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
    EventStreamWriterMock<TestEvent> writer = new EventStreamWriterMock<>();
    CheckpointStore checkpointStore = CheckpointStoreFactory.createInMemoryStore();
    CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.None).build();
    EventProcessorGroupConfig config = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(READER_GROUP).streamName(STREAM_NAME).checkpointConfig(checkpointConfig).build();
    createEventProcessorGroupConfig(initialCount);
    EventProcessorSystemImpl system = createMockSystem(systemName, PROCESS, SCOPE, createEventReaders(1, input), writer, readerGroupName);
    EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new StartWritingEventProcessor(false, input)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
    // Create EventProcessorGroup.
    EventProcessorGroupImpl<TestEvent> group = (EventProcessorGroupImpl<TestEvent>) system.createEventProcessorGroup(eventProcessorConfig, checkpointStore);
    // Await until it is ready.
    group.awaitRunning();
    // By now, the events have been written to the Mock EventStreamWriter.
    Integer[] writerList = writer.getEventList().stream().map(TestEvent::getNumber).collect(Collectors.toList()).toArray(new Integer[input.length]);
    // Validate that events are correctly written.
    Assert.assertArrayEquals(input, ArrayUtils.toPrimitive(writerList));
}
Also used : EventStreamWriterMock(io.pravega.controller.mocks.EventStreamWriterMock) CheckpointConfig(io.pravega.controller.eventProcessor.CheckpointConfig) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) CheckpointStore(io.pravega.controller.store.checkpoint.CheckpointStore) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) EventProcessorGroupConfig(io.pravega.controller.eventProcessor.EventProcessorGroupConfig) Test(org.junit.Test)

Example 7 with EventProcessorGroupConfig

use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.

the class EventProcessorTest method testInitialize.

@Test(timeout = 10000)
public void testInitialize() throws ReinitializationRequiredException, CheckpointStoreException {
    String systemName = "testSystem";
    String readerGroupName = "testReaderGroup";
    EventStreamWriterMock<TestEvent> writer = new EventStreamWriterMock<>();
    int[] input = { 1, 2, 3, 4, 5 };
    CheckpointStore checkpointStore = CheckpointStoreFactory.createInMemoryStore();
    CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.None).build();
    EventProcessorGroupConfig config = EventProcessorGroupConfigImpl.builder().eventProcessorCount(3).readerGroupName(READER_GROUP).streamName(STREAM_NAME).checkpointConfig(checkpointConfig).build();
    createEventProcessorGroupConfig(3);
    EventProcessorSystemImpl system = createMockSystem(systemName, PROCESS, SCOPE, createEventReaders(3, input), writer, readerGroupName);
    EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new StartWritingEventProcessor(false, input)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
    // Create EventProcessorGroup.
    EventProcessorGroupImpl<TestEvent> group = (EventProcessorGroupImpl<TestEvent>) system.createEventProcessorGroup(eventProcessorConfig, checkpointStore);
    // test idempotent initialize
    group.initialize();
    group.initialize();
    // Await until it is ready.
    group.awaitRunning();
    group.initialize();
    assertEquals(3, group.getEventProcessorMap().values().size());
}
Also used : EventStreamWriterMock(io.pravega.controller.mocks.EventStreamWriterMock) CheckpointConfig(io.pravega.controller.eventProcessor.CheckpointConfig) ArgumentMatchers.anyString(org.mockito.ArgumentMatchers.anyString) CheckpointStore(io.pravega.controller.store.checkpoint.CheckpointStore) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) EventProcessorGroupConfig(io.pravega.controller.eventProcessor.EventProcessorGroupConfig) Test(org.junit.Test)

Example 8 with EventProcessorGroupConfig

use of io.pravega.controller.eventProcessor.EventProcessorGroupConfig in project pravega by pravega.

the class ControllerEventProcessors method initialize.

private void initialize() throws Exception {
    // region Create commit event processor
    EventProcessorGroupConfig commitReadersConfig = EventProcessorGroupConfigImpl.builder().streamName(config.getCommitStreamName()).readerGroupName(config.getCommitReaderGroupName()).eventProcessorCount(config.getCommitReaderGroupSize()).checkpointConfig(config.getCommitCheckpointConfig()).build();
    EventProcessorConfig<CommitEvent> commitConfig = EventProcessorConfig.<CommitEvent>builder().config(commitReadersConfig).decider(ExceptionHandler.DEFAULT_EXCEPTION_HANDLER).serializer(COMMIT_EVENT_SERIALIZER).supplier(() -> this.commitEventProcessor).build();
    log.info("Creating commit event processors");
    Retry.indefinitelyWithExpBackoff(DELAY, MULTIPLIER, MAX_DELAY, e -> log.warn("Error creating commit event processor group", e)).run(() -> {
        commitEventProcessors = system.createEventProcessorGroup(commitConfig, checkpointStore);
        return null;
    });
    // endregion
    // region Create abort event processor
    EventProcessorGroupConfig abortReadersConfig = EventProcessorGroupConfigImpl.builder().streamName(config.getAbortStreamName()).readerGroupName(config.getAbortReaderGroupName()).eventProcessorCount(config.getAbortReaderGroupSize()).checkpointConfig(config.getAbortCheckpointConfig()).build();
    EventProcessorConfig<AbortEvent> abortConfig = EventProcessorConfig.<AbortEvent>builder().config(abortReadersConfig).decider(ExceptionHandler.DEFAULT_EXCEPTION_HANDLER).serializer(ABORT_EVENT_SERIALIZER).supplier(() -> new ConcurrentEventProcessor<>(abortRequestHandler, executor)).build();
    log.info("Creating abort event processors");
    Retry.indefinitelyWithExpBackoff(DELAY, MULTIPLIER, MAX_DELAY, e -> log.warn("Error creating commit event processor group", e)).run(() -> {
        abortEventProcessors = system.createEventProcessorGroup(abortConfig, checkpointStore);
        return null;
    });
    // endregion
    // region Create request event processor
    EventProcessorGroupConfig requestReadersConfig = EventProcessorGroupConfigImpl.builder().streamName(config.getRequestStreamName()).readerGroupName(config.getRequestReaderGroupName()).eventProcessorCount(1).checkpointConfig(config.getRequestStreamCheckpointConfig()).build();
    EventProcessorConfig<ControllerEvent> requestConfig = EventProcessorConfig.builder().config(requestReadersConfig).decider(ExceptionHandler.DEFAULT_EXCEPTION_HANDLER).serializer(CONTROLLER_EVENT_SERIALIZER).supplier(() -> new ConcurrentEventProcessor<>(streamRequestHandler, executor)).build();
    log.info("Creating request event processors");
    Retry.indefinitelyWithExpBackoff(DELAY, MULTIPLIER, MAX_DELAY, e -> log.warn("Error creating request event processor group", e)).run(() -> {
        requestEventProcessors = system.createEventProcessorGroup(requestConfig, checkpointStore);
        return null;
    });
    // endregion
    log.info("Awaiting start of commit event processors");
    commitEventProcessors.awaitRunning();
    log.info("Awaiting start of abort event processors");
    abortEventProcessors.awaitRunning();
    log.info("Awaiting start of request event processors");
    requestEventProcessors.awaitRunning();
}
Also used : CommitEvent(io.pravega.shared.controller.event.CommitEvent) Retry(io.pravega.common.util.Retry) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) CheckpointStore(io.pravega.controller.store.checkpoint.CheckpointStore) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) CheckpointStoreException(io.pravega.controller.store.checkpoint.CheckpointStoreException) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) AbortRequestHandler(io.pravega.controller.server.eventProcessor.requesthandlers.AbortRequestHandler) AutoScaleTask(io.pravega.controller.server.eventProcessor.requesthandlers.AutoScaleTask) RETRYABLE_PREDICATE(io.pravega.controller.util.RetryHelper.RETRYABLE_PREDICATE) ExceptionHandler(io.pravega.controller.eventProcessor.ExceptionHandler) RetryHelper.withRetriesAsync(io.pravega.controller.util.RetryHelper.withRetriesAsync) EventProcessorConfig(io.pravega.controller.eventProcessor.EventProcessorConfig) SealStreamTask(io.pravega.controller.server.eventProcessor.requesthandlers.SealStreamTask) Set(java.util.Set) CompletionException(java.util.concurrent.CompletionException) ScaleOperationTask(io.pravega.controller.server.eventProcessor.requesthandlers.ScaleOperationTask) ControllerEvent(io.pravega.shared.controller.event.ControllerEvent) Slf4j(lombok.extern.slf4j.Slf4j) List(java.util.List) ConcurrentEventProcessor(io.pravega.controller.eventProcessor.impl.ConcurrentEventProcessor) EventProcessorSystemImpl(io.pravega.controller.eventProcessor.impl.EventProcessorSystemImpl) Config(io.pravega.controller.util.Config) UpdateStreamTask(io.pravega.controller.server.eventProcessor.requesthandlers.UpdateStreamTask) ClientFactory(io.pravega.client.ClientFactory) Controller(io.pravega.client.stream.impl.Controller) StreamMetadataStore(io.pravega.controller.store.stream.StreamMetadataStore) Futures(io.pravega.common.concurrent.Futures) TruncateStreamTask(io.pravega.controller.server.eventProcessor.requesthandlers.TruncateStreamTask) SegmentHelper(io.pravega.controller.server.SegmentHelper) EventProcessorGroupConfigImpl(io.pravega.controller.eventProcessor.impl.EventProcessorGroupConfigImpl) CompletableFuture(java.util.concurrent.CompletableFuture) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) AbortEvent(io.pravega.shared.controller.event.AbortEvent) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) StreamMetadataTasks(io.pravega.controller.task.Stream.StreamMetadataTasks) ConnectionFactory(io.pravega.client.netty.impl.ConnectionFactory) Serializer(io.pravega.client.stream.Serializer) LoggerHelpers(io.pravega.common.LoggerHelpers) FailoverSweeper(io.pravega.controller.fault.FailoverSweeper) ImmutablePair(org.apache.commons.lang3.tuple.ImmutablePair) EventProcessorGroupConfig(io.pravega.controller.eventProcessor.EventProcessorGroupConfig) EventProcessorSystem(io.pravega.controller.eventProcessor.EventProcessorSystem) HostControllerStore(io.pravega.controller.store.host.HostControllerStore) StreamTransactionMetadataTasks(io.pravega.controller.task.Stream.StreamTransactionMetadataTasks) DeleteStreamTask(io.pravega.controller.server.eventProcessor.requesthandlers.DeleteStreamTask) EventProcessorGroup(io.pravega.controller.eventProcessor.EventProcessorGroup) StreamRequestHandler(io.pravega.controller.server.eventProcessor.requesthandlers.StreamRequestHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) ScalingPolicy(io.pravega.client.stream.ScalingPolicy) EventProcessorGroupConfig(io.pravega.controller.eventProcessor.EventProcessorGroupConfig) CommitEvent(io.pravega.shared.controller.event.CommitEvent) AbortEvent(io.pravega.shared.controller.event.AbortEvent) ConcurrentEventProcessor(io.pravega.controller.eventProcessor.impl.ConcurrentEventProcessor) ControllerEvent(io.pravega.shared.controller.event.ControllerEvent)

Aggregations

EventProcessorGroupConfig (io.pravega.controller.eventProcessor.EventProcessorGroupConfig)8 JavaSerializer (io.pravega.client.stream.impl.JavaSerializer)7 CheckpointStore (io.pravega.controller.store.checkpoint.CheckpointStore)6 Test (org.junit.Test)6 CheckpointConfig (io.pravega.controller.eventProcessor.CheckpointConfig)4 ArgumentMatchers.anyString (org.mockito.ArgumentMatchers.anyString)4 ClientFactory (io.pravega.client.ClientFactory)3 EventProcessorSystem (io.pravega.controller.eventProcessor.EventProcessorSystem)3 EventProcessorSystemImpl (io.pravega.controller.eventProcessor.impl.EventProcessorSystemImpl)3 ReaderGroupManagerImpl (io.pravega.client.admin.impl.ReaderGroupManagerImpl)2 StreamConfiguration (io.pravega.client.stream.StreamConfiguration)2 ClientFactoryImpl (io.pravega.client.stream.impl.ClientFactoryImpl)2 Controller (io.pravega.client.stream.impl.Controller)2 EventStreamWriterMock (io.pravega.controller.mocks.EventStreamWriterMock)2 VisibleForTesting (com.google.common.annotations.VisibleForTesting)1 AbstractIdleService (com.google.common.util.concurrent.AbstractIdleService)1 ReaderGroupManager (io.pravega.client.admin.ReaderGroupManager)1 ConnectionFactory (io.pravega.client.netty.impl.ConnectionFactory)1 ConnectionFactoryImpl (io.pravega.client.netty.impl.ConnectionFactoryImpl)1 ReaderGroup (io.pravega.client.stream.ReaderGroup)1