Search in sources :

Example 1 with EventProcessorSystem

use of io.pravega.controller.eventProcessor.EventProcessorSystem in project pravega by pravega.

the class ControllerEventProcessorsTest method testHandleOrphaned.

@Test(timeout = 10000)
public void testHandleOrphaned() {
    Controller localController = mock(Controller.class);
    CheckpointStore checkpointStore = mock(CheckpointStore.class);
    StreamMetadataStore streamStore = mock(StreamMetadataStore.class);
    HostControllerStore hostStore = mock(HostControllerStore.class);
    ConnectionFactory connectionFactory = mock(ConnectionFactory.class);
    StreamMetadataTasks streamMetadataTasks = mock(StreamMetadataTasks.class);
    StreamTransactionMetadataTasks streamTransactionMetadataTasks = mock(StreamTransactionMetadataTasks.class);
    executor = Executors.newSingleThreadScheduledExecutor();
    ControllerEventProcessorConfig config = ControllerEventProcessorConfigImpl.withDefault();
    EventProcessorSystem system = mock(EventProcessorSystem.class);
    EventProcessorGroup<ControllerEvent> processor = new EventProcessorGroup<ControllerEvent>() {

        @Override
        public void notifyProcessFailure(String process) throws CheckpointStoreException {
        }

        @Override
        public EventStreamWriter<ControllerEvent> getWriter() {
            return null;
        }

        @Override
        public Set<String> getProcesses() throws CheckpointStoreException {
            return Sets.newHashSet("host1", "host2");
        }

        @Override
        public Service startAsync() {
            return null;
        }

        @Override
        public boolean isRunning() {
            return false;
        }

        @Override
        public State state() {
            return null;
        }

        @Override
        public Service stopAsync() {
            return null;
        }

        @Override
        public void awaitRunning() {
        }

        @Override
        public void awaitRunning(long timeout, TimeUnit unit) throws TimeoutException {
        }

        @Override
        public void awaitTerminated() {
        }

        @Override
        public void awaitTerminated(long timeout, TimeUnit unit) throws TimeoutException {
        }

        @Override
        public Throwable failureCause() {
            return null;
        }

        @Override
        public void addListener(Listener listener, Executor executor) {
        }

        @Override
        public void close() throws Exception {
        }
    };
    try {
        when(system.createEventProcessorGroup(any(), any())).thenReturn(processor);
    } catch (CheckpointStoreException e) {
        e.printStackTrace();
    }
    ControllerEventProcessors processors = new ControllerEventProcessors("host1", config, localController, checkpointStore, streamStore, hostStore, SegmentHelperMock.getSegmentHelperMock(), connectionFactory, streamMetadataTasks, system, executor);
    processors.startAsync();
    processors.awaitRunning();
    assertTrue(Futures.await(processors.sweepFailedProcesses(() -> Sets.newHashSet("host1"))));
    assertTrue(Futures.await(processors.handleFailedProcess("host1")));
    processors.shutDown();
}
Also used : EventProcessorSystem(io.pravega.controller.eventProcessor.EventProcessorSystem) EventProcessorGroup(io.pravega.controller.eventProcessor.EventProcessorGroup) CheckpointStore(io.pravega.controller.store.checkpoint.CheckpointStore) Controller(io.pravega.client.stream.impl.Controller) StreamMetadataStore(io.pravega.controller.store.stream.StreamMetadataStore) ControllerEvent(io.pravega.shared.controller.event.ControllerEvent) ConnectionFactory(io.pravega.client.netty.impl.ConnectionFactory) Executor(java.util.concurrent.Executor) HostControllerStore(io.pravega.controller.store.host.HostControllerStore) StreamTransactionMetadataTasks(io.pravega.controller.task.Stream.StreamTransactionMetadataTasks) TimeUnit(java.util.concurrent.TimeUnit) CheckpointStoreException(io.pravega.controller.store.checkpoint.CheckpointStoreException) StreamMetadataTasks(io.pravega.controller.task.Stream.StreamMetadataTasks) Test(org.junit.Test)

Example 2 with EventProcessorSystem

use of io.pravega.controller.eventProcessor.EventProcessorSystem in project pravega by pravega.

the class EventProcessorTest method testEventProcessorCell.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testEventProcessorCell() throws CheckpointStoreException, ReinitializationRequiredException {
    CheckpointStore checkpointStore = CheckpointStoreFactory.createInMemoryStore();
    CheckpointConfig.CheckpointPeriod period = CheckpointConfig.CheckpointPeriod.builder().numEvents(1).numSeconds(1).build();
    CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.Periodic).checkpointPeriod(period).build();
    EventProcessorGroupConfig config = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(READER_GROUP).streamName(STREAM_NAME).checkpointConfig(checkpointConfig).build();
    int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
    int expectedSum = input.length * (input.length + 1) / 2;
    List<MockEventRead<TestEvent>> inputEvents = new ArrayList<>(input.length);
    for (int i = 0; i < input.length; i++) {
        inputEvents.add(new MockEventRead<>(i, new TestEvent(input[i])));
    }
    inputEvents.add(new MockEventRead<>(input.length, new TestEvent(-1)));
    EventProcessorSystem system = Mockito.mock(EventProcessorSystem.class);
    Mockito.when(system.getProcess()).thenReturn(PROCESS);
    EventStreamReader<TestEvent> reader = Mockito.mock(EventStreamReader.class);
    checkpointStore.addReaderGroup(PROCESS, READER_GROUP);
    // Test case 1. Actor does not throw any exception during normal operation.
    Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
    EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
    testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, expectedSum);
    // Test case 2. Actor throws an error during normal operation, and Directive is to Resume on error.
    Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
    eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(true)).serializer(new JavaSerializer<>()).decider((Throwable e) -> (e instanceof IllegalArgumentException) ? ExceptionHandler.Directive.Resume : ExceptionHandler.Directive.Stop).config(config).build();
    testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, expectedSum);
    // Test case 3. Actor throws an error during normal operation, and Directive is to Restart on error.
    Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
    eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(true)).serializer(new JavaSerializer<>()).decider((Throwable e) -> (e instanceof IllegalArgumentException) ? ExceptionHandler.Directive.Restart : ExceptionHandler.Directive.Stop).config(config).build();
    testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, 0);
    // Test case 3. Actor throws an error during normal operation, and Directive is to Restart on error.
    Mockito.when(reader.readNextEvent(anyLong())).thenAnswer(new SequenceAnswer<>(inputEvents));
    eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new RestartFailingEventProcessor(true)).serializer(new JavaSerializer<>()).decider((Throwable e) -> (e instanceof IllegalArgumentException) ? ExceptionHandler.Directive.Restart : ExceptionHandler.Directive.Stop).config(config).build();
    testEventProcessor(system, eventProcessorConfig, reader, READER_ID, checkpointStore, 3);
    // Test case 5. startup fails for an event processor
    eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(StartFailingEventProcessor::new).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(config).build();
    checkpointStore.addReader(PROCESS, READER_GROUP, READER_ID);
    EventProcessorCell<TestEvent> cell = new EventProcessorCell<>(eventProcessorConfig, reader, new EventStreamWriterMock<>(), system.getProcess(), READER_ID, 0, checkpointStore);
    cell.startAsync();
    cell.awaitTerminated();
    Assert.assertTrue(true);
}
Also used : EventProcessorSystem(io.pravega.controller.eventProcessor.EventProcessorSystem) CheckpointConfig(io.pravega.controller.eventProcessor.CheckpointConfig) ArrayList(java.util.ArrayList) CheckpointStore(io.pravega.controller.store.checkpoint.CheckpointStore) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) EventProcessorGroupConfig(io.pravega.controller.eventProcessor.EventProcessorGroupConfig) Test(org.junit.Test)

Example 3 with EventProcessorSystem

use of io.pravega.controller.eventProcessor.EventProcessorSystem in project pravega by pravega.

the class EventProcessorTest method testEventProcessor.

@Test(timeout = 60000)
public void testEventProcessor() throws Exception {
    @Cleanup TestingServer zkTestServer = new TestingServerStarter().start();
    ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig());
    serviceBuilder.initialize();
    StreamSegmentStore store = serviceBuilder.createStreamSegmentService();
    int servicePort = TestUtils.getAvailableListenPort();
    @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, servicePort, store);
    server.startListening();
    int controllerPort = TestUtils.getAvailableListenPort();
    @Cleanup ControllerWrapper controllerWrapper = new ControllerWrapper(zkTestServer.getConnectString(), true, controllerPort, "localhost", servicePort, 4);
    controllerWrapper.awaitRunning();
    Controller controller = controllerWrapper.getController();
    // Create controller object for testing against a separate controller process.
    // ControllerImpl controller = new ControllerImpl("localhost", 9090);
    final String host = "host";
    final String scope = "controllerScope";
    final String streamName = "stream1";
    final String readerGroup = "readerGroup";
    final CompletableFuture<Boolean> createScopeStatus = controller.createScope(scope);
    if (!createScopeStatus.join()) {
        throw new RuntimeException("Scope already existed");
    }
    final StreamConfiguration config = StreamConfiguration.builder().scope(scope).streamName(streamName).scalingPolicy(ScalingPolicy.fixed(1)).build();
    System.err.println(String.format("Creating stream (%s, %s)", scope, streamName));
    CompletableFuture<Boolean> createStatus = controller.createStream(config);
    if (!createStatus.get()) {
        System.err.println("Stream alrady existed, exiting");
        return;
    }
    @Cleanup ConnectionFactoryImpl connectionFactory = new ConnectionFactoryImpl(ClientConfig.builder().build());
    @Cleanup ClientFactory clientFactory = new ClientFactoryImpl(scope, controller, connectionFactory);
    @Cleanup EventStreamWriter<TestEvent> producer = clientFactory.createEventWriter(streamName, new JavaSerializer<>(), EventWriterConfig.builder().build());
    int[] input = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
    int expectedSum = input.length * (input.length + 1) / 2;
    for (int i = 0; i < input.length; i++) {
        producer.writeEvent("key", new TestEvent(input[i]));
    }
    producer.writeEvent("key", new TestEvent(-1));
    producer.flush();
    EventProcessorSystem system = new EventProcessorSystemImpl("Controller", host, scope, new ClientFactoryImpl(scope, controller, connectionFactory), new ReaderGroupManagerImpl(scope, controller, clientFactory, connectionFactory));
    CheckpointConfig.CheckpointPeriod period = CheckpointConfig.CheckpointPeriod.builder().numEvents(1).numSeconds(1).build();
    CheckpointConfig checkpointConfig = CheckpointConfig.builder().type(CheckpointConfig.Type.Periodic).checkpointPeriod(period).build();
    EventProcessorGroupConfig eventProcessorGroupConfig = EventProcessorGroupConfigImpl.builder().eventProcessorCount(1).readerGroupName(readerGroup).streamName(streamName).checkpointConfig(checkpointConfig).build();
    CompletableFuture<Long> result = new CompletableFuture<>();
    // Test case 1. Actor does not throw any exception during normal operation.
    EventProcessorConfig<TestEvent> eventProcessorConfig = EventProcessorConfig.<TestEvent>builder().supplier(() -> new TestEventProcessor(false, result)).serializer(new JavaSerializer<>()).decider((Throwable e) -> ExceptionHandler.Directive.Stop).config(eventProcessorGroupConfig).build();
    @Cleanup EventProcessorGroup<TestEvent> eventProcessorGroup = system.createEventProcessorGroup(eventProcessorConfig, CheckpointStoreFactory.createInMemoryStore());
    Long value = result.join();
    Assert.assertEquals(expectedSum, value.longValue());
    log.info("SUCCESS: received expected sum = " + expectedSum);
}
Also used : EventProcessorSystem(io.pravega.controller.eventProcessor.EventProcessorSystem) ClientFactory(io.pravega.client.ClientFactory) Cleanup(lombok.Cleanup) PravegaConnectionListener(io.pravega.segmentstore.server.host.handler.PravegaConnectionListener) JavaSerializer(io.pravega.client.stream.impl.JavaSerializer) ServiceBuilder(io.pravega.segmentstore.server.store.ServiceBuilder) ClientFactoryImpl(io.pravega.client.stream.impl.ClientFactoryImpl) CompletableFuture(java.util.concurrent.CompletableFuture) StreamConfiguration(io.pravega.client.stream.StreamConfiguration) ReaderGroupManagerImpl(io.pravega.client.admin.impl.ReaderGroupManagerImpl) ControllerWrapper(io.pravega.test.integration.demo.ControllerWrapper) TestingServer(org.apache.curator.test.TestingServer) TestingServerStarter(io.pravega.test.common.TestingServerStarter) CheckpointConfig(io.pravega.controller.eventProcessor.CheckpointConfig) Controller(io.pravega.client.stream.impl.Controller) EventProcessorSystemImpl(io.pravega.controller.eventProcessor.impl.EventProcessorSystemImpl) StreamSegmentStore(io.pravega.segmentstore.contracts.StreamSegmentStore) EventProcessorGroupConfig(io.pravega.controller.eventProcessor.EventProcessorGroupConfig) ConnectionFactoryImpl(io.pravega.client.netty.impl.ConnectionFactoryImpl) Test(org.junit.Test)

Aggregations

EventProcessorSystem (io.pravega.controller.eventProcessor.EventProcessorSystem)3 Test (org.junit.Test)3 Controller (io.pravega.client.stream.impl.Controller)2 JavaSerializer (io.pravega.client.stream.impl.JavaSerializer)2 CheckpointConfig (io.pravega.controller.eventProcessor.CheckpointConfig)2 EventProcessorGroupConfig (io.pravega.controller.eventProcessor.EventProcessorGroupConfig)2 CheckpointStore (io.pravega.controller.store.checkpoint.CheckpointStore)2 ClientFactory (io.pravega.client.ClientFactory)1 ReaderGroupManagerImpl (io.pravega.client.admin.impl.ReaderGroupManagerImpl)1 ConnectionFactory (io.pravega.client.netty.impl.ConnectionFactory)1 ConnectionFactoryImpl (io.pravega.client.netty.impl.ConnectionFactoryImpl)1 StreamConfiguration (io.pravega.client.stream.StreamConfiguration)1 ClientFactoryImpl (io.pravega.client.stream.impl.ClientFactoryImpl)1 EventProcessorGroup (io.pravega.controller.eventProcessor.EventProcessorGroup)1 EventProcessorSystemImpl (io.pravega.controller.eventProcessor.impl.EventProcessorSystemImpl)1 CheckpointStoreException (io.pravega.controller.store.checkpoint.CheckpointStoreException)1 HostControllerStore (io.pravega.controller.store.host.HostControllerStore)1 StreamMetadataStore (io.pravega.controller.store.stream.StreamMetadataStore)1 StreamMetadataTasks (io.pravega.controller.task.Stream.StreamMetadataTasks)1 StreamTransactionMetadataTasks (io.pravega.controller.task.Stream.StreamTransactionMetadataTasks)1