Examples with StreamProcessor org.apache.samza.processor.StreamProcessor used on opensource projects

Example 1 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestZkStreamProcessor method testStreamProcessor.

// main test method for happy path with fixed number of processors
private void testStreamProcessor(String[] processorIds) {
    // create a latch of the size equals to the number of messages
    TestZkStreamProcessorBase.TestStreamTask.endLatch = new CountDownLatch(messageCount);
    // initialize the processors
    StreamProcessor[] streamProcessors = new StreamProcessor[processorIds.length];
    // we need to know when the processor has started
    Object[] startWait = new Object[processorIds.length];
    for (int i = 0; i < processorIds.length; i++) {
        startWait[i] = new Object();
        streamProcessors[i] = createStreamProcessor(processorIds[i], map, startWait[i], null);
    }
    // produce messageCount messages, starting with key 0
    produceMessages(0, inputTopic, messageCount);
    // run the processors in separate threads
    Thread[] threads = new Thread[processorIds.length];
    for (int i = 0; i < processorIds.length; i++) {
        threads[i] = runInThread(streamProcessors[i], TestZkStreamProcessorBase.TestStreamTask.endLatch);
        threads[i].start();
        // wait until the processor reports that it has started
        try {
            synchronized (startWait[i]) {
                startWait[i].wait(1000);
            }
        } catch (InterruptedException e) {
            Assert.fail("got interrupted while waiting for the " + i + "th processor to start.");
        }
    }
    // wait until all the events are consumed
    try {
        TestZkStreamProcessorBase.TestStreamTask.endLatch.await(10, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        Assert.fail("endLatch.await failed with an interruption:" + e.getLocalizedMessage());
    }
    // collect all the threads
    try {
        for (Thread t : threads) {
            stopProcessor(t);
            t.join(1000);
        }
    } catch (InterruptedException e) {
        Assert.fail("Failed to join finished thread:" + e.getLocalizedMessage());
    }
    verifyNumMessages(outputTopic, messageCount, messageCount);
}

Example 2 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestZkStreamProcessor method testStreamProcessorWithAdd.

@Test
public /**
   * Similar to the previous tests, but add another processor in the middle
   */
void testStreamProcessorWithAdd() {
    // set number of events we expect to read by both processes in total:
    // p1 - reads 'messageCount' at first
    // p1 and p2 read all messageCount together, since they start from the beginning.
    // so we expect total 3 x messageCounts
    int totalEventsToGenerate = 3 * messageCount;
    TestStreamTask.endLatch = new CountDownLatch(totalEventsToGenerate);
    // create first processor
    Object startWait1 = new Object();
    Object stopWait1 = new Object();
    StreamProcessor sp = createStreamProcessor("20", map, startWait1, stopWait1);
    // produce first batch of messages starting with 0
    produceMessages(0, inputTopic, messageCount);
    // start the first processor
    Thread t1 = runInThread(sp, TestStreamTask.endLatch);
    t1.start();
    // wait until the processor reports that it has started
    waitForProcessorToStartStop(startWait1);
    // make sure it consumes all the messages from the first batch
    waitUntilMessagesLeftN(totalEventsToGenerate - messageCount);
    // start the second processor
    Object startWait2 = new Object();
    StreamProcessor sp2 = createStreamProcessor("21", map, startWait2, null);
    Thread t2 = runInThread(sp2, TestStreamTask.endLatch);
    t2.start();
    // wait until 2nd processor reports that it has started
    waitForProcessorToStartStop(startWait2);
    // wait until the 1st processor reports that it has stopped
    waitForProcessorToStartStop(stopWait1);
    // let the system to publish and distribute the new job model
    TestZkUtils.sleepMs(600);
    // produce the second batch of the messages, starting with 'messageCount'
    produceMessages(messageCount, inputTopic, messageCount);
    // wait until all the events are consumed
    waitUntilMessagesLeftN(0);
    // shutdown both
    try {
        stopProcessor(t1);
        stopProcessor(t2);
        t1.join(1000);
        t2.join(1000);
    } catch (InterruptedException e) {
        Assert.fail("Failed to join finished threads:" + e.getLocalizedMessage());
    }
    // p1 will read messageCount events, and then p1 and p2 will read 2xmessageCount events together,
    // but the expected values are the same 0-79, they will appear in the output more then once, but we should mark then only one time.
    // total number of events we gonna get is 80+40=120
    verifyNumMessages(outputTopic, 2 * messageCount, totalEventsToGenerate);
}

Example 3 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestLocalApplicationRunner method testKill.

@Test
public void testKill() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        return null;
    }).when(sp).start();
    doAnswer(new Answer() {

        private int count = 0;

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            if (++count == 1) {
                ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
                listener.afterStop();
                return null;
            }
            return null;
        }
    }).when(sp).stop();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.kill();
    verify(coordinatorStreamStore).init();
    verify(coordinatorStreamStore, atLeastOnce()).close();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}

Example 4 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestLocalApplicationRunner method testRunStreamTaskWithoutExternalContext.

@Test
public void testRunStreamTaskWithoutExternalContext() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(ApplicationConfig.APP_NAME, "test-app");
    cfgs.put(ApplicationConfig.APP_ID, "test-appId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTest();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore metadataStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.empty()), any(CoordinatorStreamStore.class));
    doReturn(metadataStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    doReturn(ApplicationStatus.SuccessfulFinish).when(runner).status();
    runner.run();
    verify(metadataStore).init();
    verify(metadataStore).close();
    assertEquals(ApplicationStatus.SuccessfulFinish, runner.status());
}

Example 5 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestLocalApplicationRunner method testRunStreamTask.

@Test
public void testRunStreamTask() throws Exception {
    final Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    cfgs.put(ApplicationConfig.APP_NAME, "test-app");
    cfgs.put(ApplicationConfig.APP_ID, "test-appId");
    config = new MapConfig(cfgs);
    mockApp = new LegacyTaskApplication(IdentityStreamTask.class.getName());
    prepareTest();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore metadataStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(metadataStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    doReturn(ApplicationStatus.SuccessfulFinish).when(runner).status();
    runner.run(externalContext);
    verify(metadataStore).init();
    verify(metadataStore).close();
    assertEquals(ApplicationStatus.SuccessfulFinish, runner.status());
}