Examples with StreamDefinition com.twitter.heron.eco.definition.StreamDefinition used on opensource projects

Example 1 with StreamDefinition

use of com.twitter.heron.eco.definition.StreamDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs.

@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.FIELDS);
    List<String> args = new ArrayList<>();
    args.add("arg1");
    groupingDefinition.setArgs(args);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIBasicBolt mockIBasicBolt = new MockIBasicBolt();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIBasicBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).fieldsGrouping(eq(from), eq(streamId), any(Fields.class));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
}

Example 2 with StreamDefinition

use of com.twitter.heron.eco.definition.StreamDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIRichBolt_ShuffleGrouping.

@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIRichBolt_ShuffleGrouping() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.SHUFFLE);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIRichBolt mockIRichBolt = new MockIRichBolt();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIRichBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).shuffleGrouping(eq(from), eq(streamId));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
}

Example 3 with StreamDefinition

use of com.twitter.heron.eco.definition.StreamDefinition in project incubator-heron by apache.

the class EcoParserTest method testParseFromInputStream_VerifyAllButComponents_MapsAsExpected.

@Test
public void testParseFromInputStream_VerifyAllButComponents_MapsAsExpected() throws Exception {
    InputStream inputStream = new ByteArrayInputStream(YAML_STR.getBytes());
    FileInputStream mockPropsStream = PowerMockito.mock(FileInputStream.class);
    EcoTopologyDefinition topologyDefinition = subject.parseFromInputStream(inputStream, mockPropsStream, false);
    assertEquals("yaml-topology", topologyDefinition.getName());
    assertEquals(1, topologyDefinition.getConfig().size());
    assertEquals(1, topologyDefinition.getConfig().get("topology.workers"));
    BoltDefinition bolt1 = topologyDefinition.getBolt(BOLT_1);
    assertNotNull(bolt1);
    assertEquals(2, bolt1.getParallelism());
    assertEquals("com.twitter.heron.sample.TestWordCounter", bolt1.getClassName());
    assertEquals(BOLT_1, bolt1.getId());
    BoltDefinition bolt2 = topologyDefinition.getBolt(BOLT_2);
    assertEquals(1, bolt2.getParallelism());
    assertEquals("com.twitter.heron.sample.LogInfoBolt", bolt2.getClassName());
    assertEquals(BOLT_2, bolt2.getId());
    List<StreamDefinition> streamDefinitions = topologyDefinition.getStreams();
    StreamDefinition streamDefinitionOne = streamDefinitions.get(0);
    GroupingDefinition groupingDefinitionOne = streamDefinitionOne.getGrouping();
    StreamDefinition streamDefinitionTwo = streamDefinitions.get(1);
    GroupingDefinition groupingDefinitionTwo = streamDefinitionTwo.getGrouping();
    assertEquals(2, streamDefinitions.size());
    assertEquals(BOLT_1, streamDefinitionOne.getTo());
    assertEquals("spout-1", streamDefinitionOne.getFrom());
    assertEquals(GroupingDefinition.Type.FIELDS, groupingDefinitionOne.getType());
    assertEquals(1, groupingDefinitionOne.getArgs().size());
    assertEquals("word", groupingDefinitionOne.getArgs().get(0));
    assertEquals("connection-1", streamDefinitionOne.getId());
    assertEquals(BOLT_2, streamDefinitionTwo.getTo());
    assertEquals("bolt-1", streamDefinitionTwo.getFrom());
    assertEquals(GroupingDefinition.Type.SHUFFLE, groupingDefinitionTwo.getType());
    assertEquals("connection-2", streamDefinitionTwo.getId());
    assertNull(groupingDefinitionTwo.getArgs());
}

Example 4 with StreamDefinition

use of com.twitter.heron.eco.definition.StreamDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIBasicBolt_FieldsGroupingWithoutArgs_ExceptionThrown.

@Test(expected = IllegalArgumentException.class)
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIBasicBolt_FieldsGroupingWithoutArgs_ExceptionThrown() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.FIELDS);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIBasicBolt mockIBasicBolt = new MockIBasicBolt();
    try {
        when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
        when(mockContext.getBolt(eq(to))).thenReturn(mockIBasicBolt);
        when(mockDefinition.getStreams()).thenReturn(streams);
        when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
        when(mockTopologyBuilder.setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
        subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    } finally {
        verify(mockContext).getTopologyDefinition();
        verify(mockContext).getBolt(eq(to));
        verify(mockDefinition).parallelismForBolt(eq(to));
        verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism));
        verify(mockDefinition).getStreams();
    }
}

Example 5 with StreamDefinition

use of com.twitter.heron.eco.definition.StreamDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIWindowedBolt_CustomGrouping.

@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIWindowedBolt_CustomGrouping() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.CUSTOM);
    MockCustomObjectDefinition mockCustomObjectDefinition = new MockCustomObjectDefinition();
    groupingDefinition.setCustomClass(mockCustomObjectDefinition);
    List<String> args = new ArrayList<>();
    args.add("arg1");
    groupingDefinition.setArgs(args);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIWindowedBolt mockIWindowedBolt = new MockIWindowedBolt();
    MockCustomStreamGrouping mockCustomStreamGrouping = new MockCustomStreamGrouping();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIWindowedBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIWindowedBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    when(mockObjectBuilder.buildObject(eq(mockCustomObjectDefinition), eq(mockContext))).thenReturn(mockCustomStreamGrouping);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIWindowedBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).customGrouping(eq(from), eq(streamId), eq(mockCustomStreamGrouping));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
    verify(mockObjectBuilder).buildObject(same(mockCustomObjectDefinition), same(mockContext));
}