Search in sources :

Example 1 with GroupingDefinition

use of com.twitter.heron.eco.definition.GroupingDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs.

@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.FIELDS);
    List<String> args = new ArrayList<>();
    args.add("arg1");
    groupingDefinition.setArgs(args);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIBasicBolt mockIBasicBolt = new MockIBasicBolt();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIBasicBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).fieldsGrouping(eq(from), eq(streamId), any(Fields.class));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
}
Also used : Fields(org.apache.storm.tuple.Fields) StreamDefinition(com.twitter.heron.eco.definition.StreamDefinition) ArrayList(java.util.ArrayList) GroupingDefinition(com.twitter.heron.eco.definition.GroupingDefinition) Test(org.junit.Test)

Example 2 with GroupingDefinition

use of com.twitter.heron.eco.definition.GroupingDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIRichBolt_ShuffleGrouping.

@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIRichBolt_ShuffleGrouping() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.SHUFFLE);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIRichBolt mockIRichBolt = new MockIRichBolt();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIRichBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).shuffleGrouping(eq(from), eq(streamId));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
}
Also used : StreamDefinition(com.twitter.heron.eco.definition.StreamDefinition) ArrayList(java.util.ArrayList) GroupingDefinition(com.twitter.heron.eco.definition.GroupingDefinition) Test(org.junit.Test)

Example 3 with GroupingDefinition

use of com.twitter.heron.eco.definition.GroupingDefinition in project incubator-heron by apache.

the class EcoParserTest method testParseFromInputStream_VerifyAllButComponents_MapsAsExpected.

@Test
public void testParseFromInputStream_VerifyAllButComponents_MapsAsExpected() throws Exception {
    InputStream inputStream = new ByteArrayInputStream(YAML_STR.getBytes());
    FileInputStream mockPropsStream = PowerMockito.mock(FileInputStream.class);
    EcoTopologyDefinition topologyDefinition = subject.parseFromInputStream(inputStream, mockPropsStream, false);
    assertEquals("yaml-topology", topologyDefinition.getName());
    assertEquals(1, topologyDefinition.getConfig().size());
    assertEquals(1, topologyDefinition.getConfig().get("topology.workers"));
    BoltDefinition bolt1 = topologyDefinition.getBolt(BOLT_1);
    assertNotNull(bolt1);
    assertEquals(2, bolt1.getParallelism());
    assertEquals("com.twitter.heron.sample.TestWordCounter", bolt1.getClassName());
    assertEquals(BOLT_1, bolt1.getId());
    BoltDefinition bolt2 = topologyDefinition.getBolt(BOLT_2);
    assertEquals(1, bolt2.getParallelism());
    assertEquals("com.twitter.heron.sample.LogInfoBolt", bolt2.getClassName());
    assertEquals(BOLT_2, bolt2.getId());
    List<StreamDefinition> streamDefinitions = topologyDefinition.getStreams();
    StreamDefinition streamDefinitionOne = streamDefinitions.get(0);
    GroupingDefinition groupingDefinitionOne = streamDefinitionOne.getGrouping();
    StreamDefinition streamDefinitionTwo = streamDefinitions.get(1);
    GroupingDefinition groupingDefinitionTwo = streamDefinitionTwo.getGrouping();
    assertEquals(2, streamDefinitions.size());
    assertEquals(BOLT_1, streamDefinitionOne.getTo());
    assertEquals("spout-1", streamDefinitionOne.getFrom());
    assertEquals(GroupingDefinition.Type.FIELDS, groupingDefinitionOne.getType());
    assertEquals(1, groupingDefinitionOne.getArgs().size());
    assertEquals("word", groupingDefinitionOne.getArgs().get(0));
    assertEquals("connection-1", streamDefinitionOne.getId());
    assertEquals(BOLT_2, streamDefinitionTwo.getTo());
    assertEquals("bolt-1", streamDefinitionTwo.getFrom());
    assertEquals(GroupingDefinition.Type.SHUFFLE, groupingDefinitionTwo.getType());
    assertEquals("connection-2", streamDefinitionTwo.getId());
    assertNull(groupingDefinitionTwo.getArgs());
}
Also used : BoltDefinition(com.twitter.heron.eco.definition.BoltDefinition) StreamDefinition(com.twitter.heron.eco.definition.StreamDefinition) ByteArrayInputStream(java.io.ByteArrayInputStream) FileInputStream(java.io.FileInputStream) ByteArrayInputStream(java.io.ByteArrayInputStream) InputStream(java.io.InputStream) EcoTopologyDefinition(com.twitter.heron.eco.definition.EcoTopologyDefinition) GroupingDefinition(com.twitter.heron.eco.definition.GroupingDefinition) FileInputStream(java.io.FileInputStream) Test(org.junit.Test)

Example 4 with GroupingDefinition

use of com.twitter.heron.eco.definition.GroupingDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIBasicBolt_FieldsGroupingWithoutArgs_ExceptionThrown.

@Test(expected = IllegalArgumentException.class)
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIBasicBolt_FieldsGroupingWithoutArgs_ExceptionThrown() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.FIELDS);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIBasicBolt mockIBasicBolt = new MockIBasicBolt();
    try {
        when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
        when(mockContext.getBolt(eq(to))).thenReturn(mockIBasicBolt);
        when(mockDefinition.getStreams()).thenReturn(streams);
        when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
        when(mockTopologyBuilder.setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
        subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    } finally {
        verify(mockContext).getTopologyDefinition();
        verify(mockContext).getBolt(eq(to));
        verify(mockDefinition).parallelismForBolt(eq(to));
        verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism));
        verify(mockDefinition).getStreams();
    }
}
Also used : StreamDefinition(com.twitter.heron.eco.definition.StreamDefinition) ArrayList(java.util.ArrayList) GroupingDefinition(com.twitter.heron.eco.definition.GroupingDefinition) Test(org.junit.Test)

Example 5 with GroupingDefinition

use of com.twitter.heron.eco.definition.GroupingDefinition in project incubator-heron by apache.

the class StreamBuilderTest method buildStreams_SpoutToIWindowedBolt_CustomGrouping.

@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIWindowedBolt_CustomGrouping() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.CUSTOM);
    MockCustomObjectDefinition mockCustomObjectDefinition = new MockCustomObjectDefinition();
    groupingDefinition.setCustomClass(mockCustomObjectDefinition);
    List<String> args = new ArrayList<>();
    args.add("arg1");
    groupingDefinition.setArgs(args);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIWindowedBolt mockIWindowedBolt = new MockIWindowedBolt();
    MockCustomStreamGrouping mockCustomStreamGrouping = new MockCustomStreamGrouping();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIWindowedBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIWindowedBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    when(mockObjectBuilder.buildObject(eq(mockCustomObjectDefinition), eq(mockContext))).thenReturn(mockCustomStreamGrouping);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIWindowedBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).customGrouping(eq(from), eq(streamId), eq(mockCustomStreamGrouping));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
    verify(mockObjectBuilder).buildObject(same(mockCustomObjectDefinition), same(mockContext));
}
Also used : StreamDefinition(com.twitter.heron.eco.definition.StreamDefinition) ArrayList(java.util.ArrayList) GroupingDefinition(com.twitter.heron.eco.definition.GroupingDefinition) Test(org.junit.Test)

Aggregations

GroupingDefinition (com.twitter.heron.eco.definition.GroupingDefinition)6 StreamDefinition (com.twitter.heron.eco.definition.StreamDefinition)6 Test (org.junit.Test)5 ArrayList (java.util.ArrayList)4 EcoTopologyDefinition (com.twitter.heron.eco.definition.EcoTopologyDefinition)2 Fields (org.apache.storm.tuple.Fields)2 BoltDefinition (com.twitter.heron.eco.definition.BoltDefinition)1 ComponentStream (com.twitter.heron.eco.definition.ComponentStream)1 ByteArrayInputStream (java.io.ByteArrayInputStream)1 FileInputStream (java.io.FileInputStream)1 InputStream (java.io.InputStream)1 HashMap (java.util.HashMap)1 BoltDeclarer (org.apache.storm.topology.BoltDeclarer)1 IBasicBolt (org.apache.storm.topology.IBasicBolt)1 IRichBolt (org.apache.storm.topology.IRichBolt)1 IWindowedBolt (org.apache.storm.topology.IWindowedBolt)1