Examples with StreamProcessor org.apache.samza.processor.StreamProcessor used on opensource projects

Example 6 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestLocalApplicationRunner method testKillWithoutCoordinatorStream.

@Test
public void testKillWithoutCoordinatorStream() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
        listener.afterStart();
        return null;
    }).when(sp).start();
    doAnswer(new Answer() {

        private int count = 0;

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            if (++count == 1) {
                ProcessorLifecycleListener listener = captor.getValue().createInstance(sp);
                listener.afterStop();
                return null;
            }
            return null;
        }
    }).when(sp).stop();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(null).when(runner).createCoordinatorStreamStore(any(Config.class));
    runner.run(externalContext);
    runner.kill();
    assertEquals(runner.status(), ApplicationStatus.SuccessfulFinish);
}

Example 7 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestLocalApplicationRunner method prepareTestForRunId.

private void prepareTestForRunId() throws Exception {
    coordinationUtils = mock(CoordinationUtils.class);
    DistributedLock lock = mock(DistributedLock.class);
    when(lock.lock(anyObject())).thenReturn(true);
    when(coordinationUtils.getLock(anyString())).thenReturn(lock);
    clusterMembership = mock(ClusterMembership.class);
    when(clusterMembership.getNumberOfProcessors()).thenReturn(1);
    when(coordinationUtils.getClusterMembership()).thenReturn(clusterMembership);
    metadataStore = mock(ZkMetadataStore.class);
    when(metadataStore.get(any())).thenReturn(null);
    PowerMockito.whenNew(ZkMetadataStore.class).withAnyArguments().thenReturn(metadataStore);
    ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc = ApplicationDescriptorUtil.getAppDescriptor(mockApp, config);
    runner = spy(new LocalApplicationRunner(mockApp, config, coordinationUtils));
    localPlanner = spy(new LocalJobPlanner(appDesc, coordinationUtils, "FAKE_UID", "FAKE_RUNID"));
    doReturn(localPlanner).when(runner).getPlanner();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), anyObject(), anyObject(), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
}

Example 8 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class TestLocalApplicationRunner method testRunFailure.

@Test
public void testRunFailure() throws Exception {
    Map<String, String> cfgs = new HashMap<>();
    cfgs.put(ApplicationConfig.PROCESSOR_ID, "0");
    config = new MapConfig(cfgs);
    ProcessorLifecycleListenerFactory mockFactory = (pContext, cfg) -> mock(ProcessorLifecycleListener.class);
    mockApp = (StreamApplication) appDesc -> appDesc.withProcessorLifecycleListenerFactory(mockFactory);
    prepareTest();
    // return the jobConfigs from the planner
    doReturn(Collections.singletonList(new JobConfig(new MapConfig(config)))).when(localPlanner).prepareJobs();
    StreamProcessor sp = mock(StreamProcessor.class);
    CoordinatorStreamStore coordinatorStreamStore = mock(CoordinatorStreamStore.class);
    ArgumentCaptor<StreamProcessor.StreamProcessorLifecycleListenerFactory> captor = ArgumentCaptor.forClass(StreamProcessor.StreamProcessorLifecycleListenerFactory.class);
    doAnswer(i -> {
        throw new Exception("test failure");
    }).when(sp).start();
    ExternalContext externalContext = mock(ExternalContext.class);
    doReturn(sp).when(runner).createStreamProcessor(anyObject(), anyObject(), captor.capture(), eq(Optional.of(externalContext)), any(CoordinatorStreamStore.class));
    doReturn(coordinatorStreamStore).when(runner).createCoordinatorStreamStore(any(Config.class));
    try {
        runner.run(externalContext);
        runner.waitForFinish();
    } catch (Throwable th) {
        assertNotNull(th);
    }
    verify(coordinatorStreamStore).init();
    verify(coordinatorStreamStore, never()).close();
    assertEquals(runner.status(), ApplicationStatus.UnsuccessfulFinish);
}

Example 9 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class WatermarkIntegrationTest method getTaskOperationGraphs.

Map<String, StreamOperatorTask> getTaskOperationGraphs(MockLocalApplicationRunner runner) throws Exception {
    StreamProcessor processor = runner.getProcessors().iterator().next();
    SamzaContainer container = TestStreamProcessorUtil.getContainer(processor);
    Map<TaskName, TaskInstance> taskInstances = JavaConverters.mapAsJavaMapConverter(container.getTaskInstances()).asJava();
    Map<String, StreamOperatorTask> tasks = new HashMap<>();
    for (Map.Entry<TaskName, TaskInstance> entry : taskInstances.entrySet()) {
        StreamOperatorTask task = (StreamOperatorTask) entry.getValue().task();
        tasks.put(entry.getKey().getTaskName(), task);
    }
    return tasks;
}

Example 10 with StreamProcessor

use of org.apache.samza.processor.StreamProcessor in project samza by apache.

the class LocalApplicationRunner method run.

@Override
public void run(StreamApplication app) {
    try {
        // 1. initialize and plan
        ExecutionPlan plan = getExecutionPlan(app);
        writePlanJsonFile(plan.getPlanAsJson());
        // 2. create the necessary streams
        createStreams(plan.getIntermediateStreams());
        // 3. create the StreamProcessors
        if (plan.getJobConfigs().isEmpty()) {
            throw new SamzaException("No jobs to run.");
        }
        plan.getJobConfigs().forEach(jobConfig -> {
            log.debug("Starting job {} StreamProcessor with config {}", jobConfig.getName(), jobConfig);
            LocalStreamProcessorLifeCycleListener listener = new LocalStreamProcessorLifeCycleListener();
            StreamProcessor processor = createStreamProcessor(jobConfig, app, listener);
            listener.setProcessor(processor);
            processors.add(processor);
        });
        numProcessorsToStart.set(processors.size());
        // 4. start the StreamProcessors
        processors.forEach(StreamProcessor::start);
    } catch (Exception e) {
        throw new SamzaException("Failed to start application", e);
    }
}