Examples with StreamSpec org.apache.samza.system.StreamSpec used on opensource projects

Example 26 with StreamSpec

use of org.apache.samza.system.StreamSpec in project samza by apache.

the class TestAbstractApplicationRunner method testgetStreamWithSystemAtBothScopesInConfig.

// Stream scope should override default scope
@Test
public void testgetStreamWithSystemAtBothScopesInConfig() {
    Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, StreamConfig.SYSTEM(), TEST_SYSTEM), JobConfig.JOB_DEFAULT_SYSTEM(), TEST_DEFAULT_SYSTEM);
    AbstractApplicationRunner runner = new TestAbstractApplicationRunnerImpl(config);
    StreamSpec spec = runner.getStreamSpec(STREAM_ID);
    assertEquals(TEST_SYSTEM, spec.getSystemName());
}

Example 27 with StreamSpec

use of org.apache.samza.system.StreamSpec in project samza by apache.

the class TestAbstractApplicationRunner method testgetStreamWithSystemAtStreamScopeInConfig.

// If the system is specified at the stream scope, use it
@Test
public void testgetStreamWithSystemAtStreamScopeInConfig() {
    Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, StreamConfig.SYSTEM(), TEST_SYSTEM);
    AbstractApplicationRunner runner = new TestAbstractApplicationRunnerImpl(config);
    StreamSpec spec = runner.getStreamSpec(STREAM_ID);
    assertEquals(TEST_SYSTEM, spec.getSystemName());
}

Example 28 with StreamSpec

use of org.apache.samza.system.StreamSpec in project samza by apache.

the class TestLocalApplicationRunner method testStreamCreationWithCoordination.

@Test
public void testStreamCreationWithCoordination() throws Exception {
    Map<String, String> config = new HashMap<>();
    LocalApplicationRunner runner = new LocalApplicationRunner(new MapConfig(config));
    StreamApplication app = mock(StreamApplication.class);
    doNothing().when(app).init(anyObject(), anyObject());
    ExecutionPlanner planner = mock(ExecutionPlanner.class);
    Field plannerField = runner.getClass().getSuperclass().getDeclaredField("planner");
    plannerField.setAccessible(true);
    plannerField.set(runner, planner);
    StreamManager streamManager = mock(StreamManager.class);
    Field streamManagerField = runner.getClass().getSuperclass().getDeclaredField("streamManager");
    streamManagerField.setAccessible(true);
    streamManagerField.set(runner, streamManager);
    ArgumentCaptor<List> captor = ArgumentCaptor.forClass(List.class);
    ExecutionPlan plan = new ExecutionPlan() {

        @Override
        public List<JobConfig> getJobConfigs() {
            return Collections.emptyList();
        }

        @Override
        public List<StreamSpec> getIntermediateStreams() {
            return Collections.singletonList(new StreamSpec("test-stream", "test-stream", "test-system"));
        }

        @Override
        public String getPlanAsJson() throws Exception {
            return "";
        }
    };
    when(planner.plan(anyObject())).thenReturn(plan);
    LocalApplicationRunner spy = spy(runner);
    CoordinationUtils coordinationUtils = mock(CoordinationUtils.class);
    LeaderElector leaderElector = new LeaderElector() {

        private LeaderElectorListener leaderElectorListener;

        @Override
        public void setLeaderElectorListener(LeaderElectorListener listener) {
            this.leaderElectorListener = listener;
        }

        @Override
        public void tryBecomeLeader() {
            leaderElectorListener.onBecomingLeader();
        }

        @Override
        public void resignLeadership() {
        }

        @Override
        public boolean amILeader() {
            return false;
        }
    };
    Latch latch = new Latch() {

        boolean done = false;

        @Override
        public void await(long timeout, TimeUnit tu) throws TimeoutException {
            // in this test, latch is released before wait
            assertTrue(done);
        }

        @Override
        public void countDown() {
            done = true;
        }
    };
    when(coordinationUtils.getLeaderElector()).thenReturn(leaderElector);
    when(coordinationUtils.getLatch(anyInt(), anyString())).thenReturn(latch);
    doReturn(coordinationUtils).when(spy).createCoordinationUtils();
    try {
        spy.run(app);
    } catch (Throwable t) {
        //no jobs exception
        assertNotNull(t);
    }
    verify(streamManager).createStreams(captor.capture());
    List<StreamSpec> streamSpecs = captor.getValue();
    assertEquals(streamSpecs.size(), 1);
    assertEquals(streamSpecs.get(0).getId(), "test-stream");
}

Example 29 with StreamSpec

use of org.apache.samza.system.StreamSpec in project samza by apache.

the class TestJobGraph method testAddSource.

@Test
public void testAddSource() {
    JobGraph graph = new JobGraph(null);
    /**
     * s1 -> 1
     * s2 ->|
     *
     * s3 -> 2
     *   |-> 3
     */
    JobNode n1 = graph.getOrCreateJobNode("1", "1", null);
    JobNode n2 = graph.getOrCreateJobNode("2", "1", null);
    JobNode n3 = graph.getOrCreateJobNode("3", "1", null);
    StreamSpec s1 = genStream();
    StreamSpec s2 = genStream();
    StreamSpec s3 = genStream();
    graph.addSource(s1, n1);
    graph.addSource(s2, n1);
    graph.addSource(s3, n2);
    graph.addSource(s3, n3);
    assertTrue(graph.getSources().size() == 3);
    assertTrue(graph.getOrCreateJobNode("1", "1", null).getInEdges().size() == 2);
    assertTrue(graph.getOrCreateJobNode("2", "1", null).getInEdges().size() == 1);
    assertTrue(graph.getOrCreateJobNode("3", "1", null).getInEdges().size() == 1);
    assertTrue(graph.getOrCreateStreamEdge(s1).getSourceNodes().size() == 0);
    assertTrue(graph.getOrCreateStreamEdge(s1).getTargetNodes().size() == 1);
    assertTrue(graph.getOrCreateStreamEdge(s2).getSourceNodes().size() == 0);
    assertTrue(graph.getOrCreateStreamEdge(s2).getTargetNodes().size() == 1);
    assertTrue(graph.getOrCreateStreamEdge(s3).getSourceNodes().size() == 0);
    assertTrue(graph.getOrCreateStreamEdge(s3).getTargetNodes().size() == 2);
}

Example 30 with StreamSpec

use of org.apache.samza.system.StreamSpec in project samza by apache.

the class TestAbstractApplicationRunner method testStreamConfigOverridesWithSystemDefaults.

// Verify that we use a default specified with systems.x.default.stream.*, if specified
@Test
public void testStreamConfigOverridesWithSystemDefaults() {
    Config config = addConfigs(buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, StreamConfig.SYSTEM(), TEST_SYSTEM, "segment.bytes", "5309"), // System default property
    String.format("systems.%s.default.stream.replication.factor", TEST_SYSTEM), // System default property
    "4", String.format("systems.%s.default.stream.segment.bytest", TEST_SYSTEM), "867");
    AbstractApplicationRunner env = new TestAbstractApplicationRunnerImpl(config);
    StreamSpec spec = env.getStreamSpec(STREAM_ID);
    Map<String, String> properties = spec.getConfig();
    assertEquals(3, properties.size());
    // Uses system default
    assertEquals("4", properties.get("replication.factor"));
    // Overrides system default
    assertEquals("5309", properties.get("segment.bytes"));
}