Examples with ExecutionPlan org.apache.samza.execution.ExecutionPlan used on opensource projects

Example 1 with ExecutionPlan

use of org.apache.samza.execution.ExecutionPlan in project samza by apache.

the class RemoteApplicationRunner method status.

@Override
public ApplicationStatus status(StreamApplication app) {
    try {
        boolean hasNewJobs = false;
        boolean hasRunningJobs = false;
        ApplicationStatus unsuccessfulFinishStatus = null;
        ExecutionPlan plan = getExecutionPlan(app);
        for (JobConfig jobConfig : plan.getJobConfigs()) {
            JobRunner runner = new JobRunner(jobConfig);
            ApplicationStatus status = runner.status();
            log.debug("Status is {} for job {}", new Object[] { status, jobConfig.getName() });
            switch(status.getStatusCode()) {
                case New:
                    hasNewJobs = true;
                    break;
                case Running:
                    hasRunningJobs = true;
                    break;
                case UnsuccessfulFinish:
                    unsuccessfulFinishStatus = status;
                    break;
                case SuccessfulFinish:
                    break;
                default:
            }
        }
        if (hasNewJobs) {
            // There are jobs not started, report as New
            return ApplicationStatus.New;
        } else if (hasRunningJobs) {
            // All jobs are started, some are running
            return ApplicationStatus.Running;
        } else if (unsuccessfulFinishStatus != null) {
            // All jobs are finished, some are not successful
            return unsuccessfulFinishStatus;
        } else {
            // All jobs are finished successfully
            return ApplicationStatus.SuccessfulFinish;
        }
    } catch (Throwable t) {
        throw new SamzaException("Failed to get status for application", t);
    }
}

Example 2 with ExecutionPlan

use of org.apache.samza.execution.ExecutionPlan in project samza by apache.

the class TestLocalApplicationRunner method testStreamCreation.

@Test
public void testStreamCreation() throws Exception {
    Map<String, String> config = new HashMap<>();
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    StreamManager streamManager = mock(StreamManager.class);
    Field streamManagerField = runner.getClass().getSuperclass().getDeclaredField("streamManager");
    streamManagerField.setAccessible(true);
    streamManagerField.set(runner, streamManager);
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.emptyList();
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system"));
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    LocalApplicationRunner spy = spy(runner);
    try {
        spy.run(app);
    } catch (Throwable t) {
        //no jobs exception
        assertNotNull(t);
    }
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
}

Example 3 with ExecutionPlan

use of org.apache.samza.execution.ExecutionPlan in project samza by apache.

the class TestLocalApplicationRunner method testRunFailure.

@Test
public void testRunFailure() throws Exception {
    final Map<String, String> config = new HashMap<>();
    config.put(ApplicationConfig.PROCESSOR_ID, "0");
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.singletonList(new JobConfig(new MapConfig(config)));
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.emptyList();
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    Throwable t = new Throwable("test failure");
    StreamProcessor sp = mock(StreamProcessor.class);
    ArgumentCaptor<StreamProcessorLifecycleListener> captor = ArgumentCaptor.forClass(StreamProcessorLifecycleListener.class);
    doAnswer(i -> {
        StreamProcessorLifecycleListener listener = captor.getValue();
        listener.onFailure(t);
        return null;
    }).when(sp).start();
    LocalApplicationRunner spy = spy(runner);
    doReturn(sp).when(spy).createStreamProcessor(anyObject(), anyObject(), captor.capture());
    try {
        spy.run(app);
    } catch (Throwable th) {
        assertNotNull(th);
    }
    assertEquals(spy.status(app), ApplicationStatus.UnsuccessfulFinish);
}

Example 4 with ExecutionPlan

use of org.apache.samza.execution.ExecutionPlan in project samza by apache.

the class TestLocalApplicationRunner method testRunComplete.

@Test
public void testRunComplete() throws Exception {
    final Map<String, String> config = new HashMap<>();
    config.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.singletonList(new JobConfig(new MapConfig(config)));
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.emptyList();
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    StreamProcessor sp = mock(StreamProcessor.class);
    ArgumentCaptor<StreamProcessorLifecycleListener> captor = ArgumentCaptor.forClass(StreamProcessorLifecycleListener.class);
    doAnswer(i -> {
        StreamProcessorLifecycleListener listener = captor.getValue();
        listener.onStart();
        listener.onShutdown();
        return null;
    }).when(sp).start();
    LocalApplicationRunner spy = spy(runner);
    doReturn(sp).when(spy).createStreamProcessor(anyObject(), anyObject(), captor.capture());
    spy.run(app);
    assertEquals(spy.status(app), ApplicationStatus.SuccessfulFinish);
}

Example 5 with ExecutionPlan

use of org.apache.samza.execution.ExecutionPlan in project samza by apache.

the class TestLocalApplicationRunner method testStreamCreationWithCoordination.

@Test
public void testStreamCreationWithCoordination() throws Exception {
    Map<String, String> config = new HashMap<>();
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    StreamManager streamManager = mock(StreamManager.class);
    Field streamManagerField = runner.getClass().getSuperclass().getDeclaredField("streamManager");
    streamManagerField.setAccessible(true);
    streamManagerField.set(runner, streamManager);
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.emptyList();
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system"));
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    LocalApplicationRunner spy = spy(runner);
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    LeaderElector leaderElector = new LeaderElector() {

        private LeaderElectorListener leaderElectorListener;

        @Override
        public void setLeaderElectorListener(LeaderElectorListener listener) {
            this.leaderElectorListener = listener;
        }

        @Override
        public void tryBecomeLeader() {
            leaderElectorListener.onBecomingLeader();
        }

        @Override
        public void resignLeadership() {
        }

        @Override
        public boolean amILeader() {
            return false;
        }
    };
    Latch latch = new Latch() {

        boolean done = false;

        @Override
        public void await(long timeout, TimeUnit tu) throws TimeoutException {
            // in this test, latch is released before wait
            assertTrue(done);
        }

        @Override
        public void countDown() {
            done = true;
        }
    };
    when(coordinationUtils.getLeaderElector()).thenReturn(leaderElector);
    when(coordinationUtils.getLatch(anyInt(), anyString())).thenReturn(latch);
    doReturn(coordinationUtils).when(spy).createCoordinationUtils();
    try {
        spy.run(app);
    } catch (Throwable t) {
        //no jobs exception
        assertNotNull(t);
    }
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
}