Examples with StreamProcessor org.apache.samza.processor.StreamProcessor used on opensource projects

Example 11 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestZkStreamProcessor method testStreamProcessorWithRemove.

@Test
public /**
   * same as other happy path messages, but with one processor removed in the middle
   */
void testStreamProcessorWithRemove() {
    // set number of events we expect to read by both processes in total:
    // p1 and p2 - both read messageCount at first and p1 is shutdown, new batch of events is generated
    // and p2 will read all of them from the beginning (+ 2 x messageCounts, total 3 x)
    int totalEventsToGenerate = 3 * messageCount;
    TestStreamTask.endLatch = new CountDownLatch(totalEventsToGenerate);
    // create first processor
    Object waitStart1 = new Object();
    Object waitStop1 = new Object();
    StreamProcessor sp1 = createStreamProcessor("30", map, waitStart1, waitStop1);
    // start the first processor
    Thread t1 = runInThread(sp1, TestStreamTask.endLatch);
    t1.start();
    // start the second processor
    Object waitStart2 = new Object();
    Object waitStop2 = new Object();
    StreamProcessor sp2 = createStreamProcessor("31", map, waitStart2, waitStop2);
    Thread t2 = runInThread(sp2, TestStreamTask.endLatch);
    t2.start();
    // wait until the processor reports that it has started
    waitForProcessorToStartStop(waitStart1);
    // wait until the processor reports that it has started
    waitForProcessorToStartStop(waitStart2);
    // produce first batch of messages starting with 0
    produceMessages(0, inputTopic, messageCount);
    // make sure they consume all the messages from the first batch
    waitUntilMessagesLeftN(totalEventsToGenerate - messageCount);
    // stop the first processor
    stopProcessor(t1);
    // wait until it's really down
    waitForProcessorToStartStop(waitStop1);
    // processor1 will stop and start again. We wait for its stop to make sure we can count EXACTLY how many messages it reads.
    waitForProcessorToStartStop(waitStop2);
    // let the system to publish and distribute the new job model
    TestZkUtils.sleepMs(300);
    // produce the second batch of the messages, starting with 'messageCount'
    produceMessages(messageCount, inputTopic, messageCount);
    // wait until p2 consumes all the message by itself;
    waitUntilMessagesLeftN(0);
    try {
        stopProcessor(t2);
        t2.join(1000);
    } catch (InterruptedException e) {
        Assert.fail("Failed to join finished thread:" + e.getLocalizedMessage());
    }
    // processor1 and 2 will both read 20 events (total 40), and then processor2 read 80 events by itself,
    // but the expected values are the same 0-79 - we should get each value one time.
    // Meanwhile the number of events we gonna get is 40 + 80
    verifyNumMessages(outputTopic, 2 * messageCount, totalEventsToGenerate);
}

Example 12 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestZkStreamProcessorFailures method testFailStreamProcessor.

@Test
public // throw an exception.
void testFailStreamProcessor() {
    // either of these bad messages will cause p1 to throw and exception
    final int numBadMessages = 4;
    map.put(JobConfig.JOB_DEBOUNCE_TIME_MS(), "100");
    map.put("processor.id.to.fail", "101");
    // set number of events we expect to read by both processes in total:
    // p1 will read messageCount/2 messages
    // p2 will read messageCount/2 messages
    // numBadMessages "bad" messages will be generated
    // p2 will read 2 of the "bad" messages
    // p1 will fail on the first of the "bad" messages
    // a new job model will be generated
    // and p2 will read all 2 * messageCount messages again, + numBadMessages (all of them this time)
    // total 2 x messageCount / 2 + numBadMessages/2 + 2 * messageCount + numBadMessages
    int totalEventsToBeConsumed = 3 * messageCount;
    TestStreamTask.endLatch = new CountDownLatch(totalEventsToBeConsumed);
    // create first processor
    Object waitStart1 = new Object();
    Object waitStop1 = new Object();
    StreamProcessor sp1 = createStreamProcessor("101", map, waitStart1, waitStop1);
    // start the first processor
    Thread t1 = runInThread(sp1, TestStreamTask.endLatch);
    t1.start();
    // start the second processor
    Object waitStart2 = new Object();
    Object waitStop2 = new Object();
    StreamProcessor sp2 = createStreamProcessor("102", map, waitStart2, waitStop2);
    Thread t2 = runInThread(sp2, TestStreamTask.endLatch);
    t2.start();
    // wait until the 1st processor reports that it has started
    waitForProcessorToStartStop(waitStart1);
    // wait until the 2nd processor reports that it has started
    waitForProcessorToStartStop(waitStart2);
    // produce first batch of messages starting with 0
    produceMessages(0, inputTopic, messageCount);
    // make sure they consume all the messages
    waitUntilMessagesLeftN(totalEventsToBeConsumed - messageCount);
    // produce the bad messages
    produceMessages(BAD_MESSAGE_KEY, inputTopic, 4);
    waitForProcessorToStartStop(waitStop1);
    // wait until the 2nd processor reports that it has stopped
    waitForProcessorToStartStop(waitStop2);
    // give some extra time to let the system to publish and distribute the new job model
    TestZkUtils.sleepMs(300);
    // produce the second batch of the messages, starting with 'messageCount'
    produceMessages(messageCount, inputTopic, messageCount);
    // wait until p2 consumes all the message by itself
    waitUntilMessagesLeftN(0);
    // shutdown p2
    try {
        stopProcessor(t2);
        t2.join(1000);
    } catch (InterruptedException e) {
        Assert.fail("Failed to join finished thread:" + e.getLocalizedMessage());
    }
    // number of unique values we gonna read is from 0 to (2*messageCount - 1)
    Map<Integer, Boolean> expectedValues = new HashMap<>(2 * messageCount);
    for (int i = 0; i < 2 * messageCount; i++) {
        expectedValues.put(i, false);
    }
    for (int i = BAD_MESSAGE_KEY; i < numBadMessages + BAD_MESSAGE_KEY; i++) {
    //expectedValues.put(i, false);
    }
    verifyNumMessages(outputTopic, expectedValues, totalEventsToBeConsumed);
}

Example 13 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class LocalApplicationRunner method createStreamProcessor.

@VisibleForTesting
StreamProcessor createStreamProcessor(Config config, ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc, StreamProcessor.StreamProcessorLifecycleListenerFactory listenerFactory, Optional<ExternalContext> externalContextOptional, MetadataStore coordinatorStreamStore) {
    TaskFactory taskFactory = TaskFactoryUtil.getTaskFactory(appDesc);
    Map<String, MetricsReporter> reporters = new HashMap<>();
    String processorId = createProcessorId(new ApplicationConfig(config));
    appDesc.getMetricsReporterFactories().forEach((name, factory) -> reporters.put(name, factory.getMetricsReporter(name, processorId, config)));
    return new StreamProcessor(processorId, config, reporters, taskFactory, appDesc.getApplicationContainerContextFactory(), appDesc.getApplicationTaskContextFactory(), externalContextOptional, listenerFactory, null, coordinatorStreamStore);
}

Example 14 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class LocalApplicationRunner method run.

@Override
public void run(ExternalContext externalContext) {
    initializeRunId();
    LocalJobPlanner planner = getPlanner();
    try {
        List<JobConfig> jobConfigs = planner.prepareJobs();
        // create the StreamProcessors
        if (jobConfigs.isEmpty()) {
            throw new SamzaException("No jobs to run.");
        }
        jobConfigs.forEach(jobConfig -> {
            LOG.debug("Starting job {} StreamProcessor with config {}", jobConfig.getName(), jobConfig);
            MetadataStore coordinatorStreamStore = createCoordinatorStreamStore(jobConfig);
            if (coordinatorStreamStore != null) {
                coordinatorStreamStore.init();
            }
            StreamProcessor processor = createStreamProcessor(jobConfig, appDesc, sp -> new LocalStreamProcessorLifecycleListener(sp, jobConfig), Optional.ofNullable(externalContext), coordinatorStreamStore);
            processors.add(Pair.of(processor, coordinatorStreamStore));
        });
        numProcessorsToStart.set(processors.size());
        // start the StreamProcessors
        processors.forEach(sp -> sp.getLeft().start());
    } catch (Throwable throwable) {
        cleanup();
        appStatus = ApplicationStatus.unsuccessfulFinish(throwable);
        shutdownLatch.countDown();
        throw new SamzaException(String.format("Failed to start application: %s", new ApplicationConfig(appDesc.getConfig()).getGlobalAppId()), throwable);
    }
}

Example 15 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestLocalApplicationRunner method testRunCompleteWithouCoordinatorStreamStore.

@Test
public void testRunCompleteWithouCoordinatorStreamStore() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        listener.afterStop();
        return null;
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), eq(null));
    doReturn(null).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.waitForFinish();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}