Examples with JobConfig org.apache.samza.config.JobConfig used on opensource projects

Example 6 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalForStatelessOperators.

@Test
public void testTriggerIntervalForStatelessOperators() throws Exception {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamGraphImpl streamGraph = createSimpleGraph();
    ExecutionPlan plan = planner.plan(streamGraph);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(jobConfigs.size(), 1);
    assertFalse(jobConfigs.get(0).containsKey(TaskConfig.WINDOW_MS()));
}

Example 7 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class TestLocalApplicationRunner method testRunComplete.

@Test
public void testRunComplete() throws Exception {
    final Map<String, String> config = new HashMap<>();
    config.put(ApplicationConfig.APP_PROCESSOR_ID_GENERATOR_CLASS, UUIDGenerator.class.getName());
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.singletonList(new JobConfig(new MapConfig(config)));
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.emptyList();
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    StreamProcessor sp = mock(StreamProcessor.class);
    ArgumentCaptor<StreamProcessorLifecycleListener> captor = ArgumentCaptor.forClass(StreamProcessorLifecycleListener.class);
    doAnswer(i -> {
        StreamProcessorLifecycleListener listener = captor.getValue();
        listener.onStart();
        listener.onShutdown();
        return null;
    }).when(sp).start();
    LocalApplicationRunner spy = spy(runner);
    doReturn(sp).when(spy).createStreamProcessor(anyObject(), anyObject(), captor.capture());
    spy.run(app);
    assertEquals(spy.status(app), ApplicationStatus.SuccessfulFinish);
}

Example 8 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class TestLocalApplicationRunner method testStreamCreationWithCoordination.

@Test
public void testStreamCreationWithCoordination() throws Exception {
    Map<String, String> config = new HashMap<>();
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    StreamManager streamManager = mock(StreamManager.class);
    Field streamManagerField = runner.getClass().getSuperclass().getDeclaredField("streamManager");
    streamManagerField.setAccessible(true);
    streamManagerField.set(runner, streamManager);
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.emptyList();
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system"));
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    LocalApplicationRunner spy = spy(runner);
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    LeaderElector leaderElector = new LeaderElector() {

        private LeaderElectorListener leaderElectorListener;

        @Override
        public void setLeaderElectorListener(LeaderElectorListener listener) {
            this.leaderElectorListener = listener;
        }

        @Override
        public void tryBecomeLeader() {
            leaderElectorListener.onBecomingLeader();
        }

        @Override
        public void resignLeadership() {
        }

        @Override
        public boolean amILeader() {
            return false;
        }
    };
    Latch latch = new Latch() {

        boolean done = false;

        @Override
        public void await(long timeout, TimeUnit tu) throws TimeoutException {
            // in this test, latch is released before wait
            assertTrue(done);
        }

        @Override
        public void countDown() {
            done = true;
        }
    };
    when(coordinationUtils.getLeaderElector()).thenReturn(leaderElector);
    when(coordinationUtils.getLatch(anyInt(), anyString())).thenReturn(latch);
    doReturn(coordinationUtils).when(spy).createCoordinationUtils();
    try {
        spy.run(app);
    } catch (Throwable t) {
        //no jobs exception
        assertNotNull(t);
    }
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
}

Example 9 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalForWindowsAndJoins.

@Test
public void testTriggerIntervalForWindowsAndJoins() throws Exception {
    Map<String, String> map = new HashMap<>(config);
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamGraphImpl streamGraph = createStreamGraphWithJoinAndWindow();
    ExecutionPlan plan = planner.plan(streamGraph);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(jobConfigs.size(), 1);
    // GCD of 8, 16, 1600 and 252 is 4
    assertEquals(jobConfigs.get(0).get(TaskConfig.WINDOW_MS()), "4");
}

Example 10 with JobConfig

use of org.apache.samza.config.JobConfig in project samza by apache.

the class TestExecutionPlanner method testTriggerIntervalWhenWindowMsIsConfigured.

@Test
public void testTriggerIntervalWhenWindowMsIsConfigured() throws Exception {
    Map<String, String> map = new HashMap<>(config);
    map.put(TaskConfig.WINDOW_MS(), "2000");
    map.put(JobConfig.JOB_INTERMEDIATE_STREAM_PARTITIONS(), String.valueOf(DEFAULT_PARTITIONS));
    Config cfg = new MapConfig(map);
    ExecutionPlanner planner = new ExecutionPlanner(cfg, streamManager);
    StreamGraphImpl streamGraph = createSimpleGraph();
    ExecutionPlan plan = planner.plan(streamGraph);
    List<JobConfig> jobConfigs = plan.getJobConfigs();
    assertEquals(jobConfigs.size(), 1);
    assertEquals(jobConfigs.get(0).get(TaskConfig.WINDOW_MS()), "2000");
}