Search in sources :

Example 1 with GroupingDefinition

use of org.apache.heron.eco.definition.GroupingDefinition in project heron by twitter.

the class HeronStreamBuilderTest method buildStreams_SpoutToIStatefulWindowedBolt.

@Test
public void buildStreams_SpoutToIStatefulWindowedBolt() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.CUSTOM);
    MockCustomObjectDefinition mockCustomObjectDefinition = new MockCustomObjectDefinition();
    groupingDefinition.setCustomClass(mockCustomObjectDefinition);
    List<String> args = new ArrayList<>();
    args.add("arg1");
    groupingDefinition.setArgs(args);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIStatefulWindowedBolt mockIStatefulWindowedBolt = new MockIStatefulWindowedBolt();
    MockCustomStreamGrouping mockCustomStreamGrouping = new MockCustomStreamGrouping();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIStatefulWindowedBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIStatefulWindowedBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    when(mockObjectBuilder.buildObject(eq(mockCustomObjectDefinition), eq(mockContext))).thenReturn(mockCustomStreamGrouping);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIStatefulWindowedBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).customGrouping(eq(from), eq(streamId), eq(mockCustomStreamGrouping));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
    verify(mockObjectBuilder).buildObject(same(mockCustomObjectDefinition), same(mockContext));
}
Also used : StreamDefinition(org.apache.heron.eco.definition.StreamDefinition) ArrayList(java.util.ArrayList) GroupingDefinition(org.apache.heron.eco.definition.GroupingDefinition) Test(org.junit.Test)

Example 2 with GroupingDefinition

use of org.apache.heron.eco.definition.GroupingDefinition in project heron by twitter.

the class StormStreamBuilderTest method buildStreams_SpoutToIBasicBolt_FieldsGroupingWithoutArgs_ExceptionThrown.

@Test(expected = IllegalArgumentException.class)
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIBasicBolt_FieldsGroupingWithoutArgs_ExceptionThrown() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.FIELDS);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIBasicBolt mockIBasicBolt = new MockIBasicBolt();
    try {
        when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
        when(mockContext.getBolt(eq(to))).thenReturn(mockIBasicBolt);
        when(mockDefinition.getStreams()).thenReturn(streams);
        when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
        when(mockTopologyBuilder.setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
        subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    } finally {
        verify(mockContext).getTopologyDefinition();
        verify(mockContext).getBolt(eq(to));
        verify(mockDefinition).parallelismForBolt(eq(to));
        verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism));
        verify(mockDefinition).getStreams();
    }
}
Also used : StreamDefinition(org.apache.heron.eco.definition.StreamDefinition) ArrayList(java.util.ArrayList) GroupingDefinition(org.apache.heron.eco.definition.GroupingDefinition) Test(org.junit.Test)

Example 3 with GroupingDefinition

use of org.apache.heron.eco.definition.GroupingDefinition in project heron by twitter.

the class StormStreamBuilderTest method buildStreams_SpoutToIRichBolt_ShuffleGrouping.

@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIRichBolt_ShuffleGrouping() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
    final int iRichBoltParallelism = 1;
    final String to = "to";
    final String from = "from";
    final String streamId = "id";
    StreamDefinition streamDefinition = new StreamDefinition();
    streamDefinition.setFrom(from);
    streamDefinition.setTo(to);
    streamDefinition.setId(streamId);
    List<StreamDefinition> streams = new ArrayList<>();
    streams.add(streamDefinition);
    GroupingDefinition groupingDefinition = new GroupingDefinition();
    groupingDefinition.setType(GroupingDefinition.Type.SHUFFLE);
    groupingDefinition.setStreamId(streamId);
    streamDefinition.setGrouping(groupingDefinition);
    MockIRichBolt mockIRichBolt = new MockIRichBolt();
    when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
    when(mockContext.getBolt(eq(to))).thenReturn(mockIRichBolt);
    when(mockDefinition.getStreams()).thenReturn(streams);
    when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
    when(mockTopologyBuilder.setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
    subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
    verify(mockContext).getTopologyDefinition();
    verify(mockContext).getBolt(eq(to));
    verify(mockDefinition).parallelismForBolt(eq(to));
    verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism));
    verify(mockBoltDeclarer).shuffleGrouping(eq(from), eq(streamId));
    verify(mockContext).setStreams(anyMap());
    verify(mockDefinition).getStreams();
}
Also used : StreamDefinition(org.apache.heron.eco.definition.StreamDefinition) ArrayList(java.util.ArrayList) GroupingDefinition(org.apache.heron.eco.definition.GroupingDefinition) Test(org.junit.Test)

Example 4 with GroupingDefinition

use of org.apache.heron.eco.definition.GroupingDefinition in project heron by twitter.

the class StreamBuilder method buildStreams.

protected <K extends Serializable, V extends Serializable> void buildStreams(EcoExecutionContext executionContext, TopologyBuilder builder, ObjectBuilder objectBuilder) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchFieldException, InvocationTargetException {
    EcoTopologyDefinition topologyDefinition = executionContext.getTopologyDefinition();
    Map<String, ComponentStream> componentStreams = new HashMap<>();
    HashMap<String, BoltDeclarer> declarers = new HashMap<>();
    for (StreamDefinition stream : topologyDefinition.getStreams()) {
        Object boltObj = executionContext.getBolt(stream.getTo());
        BoltDeclarer declarer = declarers.get(stream.getTo());
        if (boltObj instanceof IRichBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IRichBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IBasicBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IBasicBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IStatefulWindowedBolt) {
            if (declarer == null) {
                // noinspection unchecked
                declarer = builder.setBolt(stream.getTo(), (IStatefulWindowedBolt<K, V>) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IWindowedBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IWindowedBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else {
            throw new IllegalArgumentException("Class does not appear to be a bolt: " + boltObj.getClass().getName());
        }
        GroupingDefinition grouping = stream.getGrouping();
        // if the streamId is defined, use it for the grouping,
        // otherwise assume default stream
        String streamId = grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId();
        switch(grouping.getType()) {
            case SHUFFLE:
                declarer.shuffleGrouping(stream.getFrom(), streamId);
                break;
            case FIELDS:
                List<String> groupingArgs = grouping.getArgs();
                if (groupingArgs == null) {
                    throw new IllegalArgumentException("You must supply arguments for Fields grouping");
                }
                declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(groupingArgs));
                break;
            case ALL:
                declarer.allGrouping(stream.getFrom(), streamId);
                break;
            case GLOBAL:
                declarer.globalGrouping(stream.getFrom(), streamId);
                break;
            case NONE:
                declarer.noneGrouping(stream.getFrom(), streamId);
                break;
            case CUSTOM:
                declarer.customGrouping(stream.getFrom(), streamId, buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), executionContext, objectBuilder));
                break;
            default:
                throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
        }
    }
    executionContext.setStreams(componentStreams);
}
Also used : IRichBolt(org.apache.heron.api.bolt.IRichBolt) ComponentStream(org.apache.heron.eco.definition.ComponentStream) StreamDefinition(org.apache.heron.eco.definition.StreamDefinition) HashMap(java.util.HashMap) IStatefulWindowedBolt(org.apache.heron.api.bolt.IStatefulWindowedBolt) EcoTopologyDefinition(org.apache.heron.eco.definition.EcoTopologyDefinition) GroupingDefinition(org.apache.heron.eco.definition.GroupingDefinition) IBasicBolt(org.apache.heron.api.bolt.IBasicBolt) Fields(org.apache.heron.api.tuple.Fields) BoltDeclarer(org.apache.heron.api.topology.BoltDeclarer) IWindowedBolt(org.apache.heron.api.bolt.IWindowedBolt)

Example 5 with GroupingDefinition

use of org.apache.heron.eco.definition.GroupingDefinition in project heron by twitter.

the class StreamBuilder method buildStreams.

protected void buildStreams(EcoExecutionContext executionContext, TopologyBuilder builder, ObjectBuilder objectBuilder) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchFieldException, InvocationTargetException {
    EcoTopologyDefinition topologyDefinition = executionContext.getTopologyDefinition();
    Map<String, ComponentStream> componentStreams = new HashMap<>();
    HashMap<String, BoltDeclarer> declarers = new HashMap<>();
    for (StreamDefinition stream : topologyDefinition.getStreams()) {
        Object boltObj = executionContext.getBolt(stream.getTo());
        BoltDeclarer declarer = declarers.get(stream.getTo());
        if (boltObj instanceof IRichBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IRichBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IBasicBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IBasicBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else if (boltObj instanceof IWindowedBolt) {
            if (declarer == null) {
                declarer = builder.setBolt(stream.getTo(), (IWindowedBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
                declarers.put(stream.getTo(), declarer);
            }
        } else {
            throw new IllegalArgumentException("Class does not appear to be a bolt: " + boltObj.getClass().getName());
        }
        GroupingDefinition grouping = stream.getGrouping();
        // if the streamId is defined, use it for the grouping,
        // otherwise assume default stream
        String streamId = grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId();
        switch(grouping.getType()) {
            case SHUFFLE:
                declarer.shuffleGrouping(stream.getFrom(), streamId);
                break;
            case FIELDS:
                List<String> groupingArgs = grouping.getArgs();
                if (groupingArgs == null) {
                    throw new IllegalArgumentException("You must supply arguments for Fields grouping");
                }
                declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(groupingArgs));
                break;
            case ALL:
                declarer.allGrouping(stream.getFrom(), streamId);
                break;
            case GLOBAL:
                declarer.globalGrouping(stream.getFrom(), streamId);
                break;
            case NONE:
                declarer.noneGrouping(stream.getFrom(), streamId);
                break;
            case CUSTOM:
                declarer.customGrouping(stream.getFrom(), streamId, buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), executionContext, objectBuilder));
                break;
            default:
                throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
        }
    }
    executionContext.setStreams(componentStreams);
}
Also used : IRichBolt(org.apache.storm.topology.IRichBolt) ComponentStream(org.apache.heron.eco.definition.ComponentStream) StreamDefinition(org.apache.heron.eco.definition.StreamDefinition) HashMap(java.util.HashMap) EcoTopologyDefinition(org.apache.heron.eco.definition.EcoTopologyDefinition) GroupingDefinition(org.apache.heron.eco.definition.GroupingDefinition) IBasicBolt(org.apache.storm.topology.IBasicBolt) Fields(org.apache.storm.tuple.Fields) BoltDeclarer(org.apache.storm.topology.BoltDeclarer) IWindowedBolt(org.apache.storm.topology.IWindowedBolt)

Aggregations

GroupingDefinition (org.apache.heron.eco.definition.GroupingDefinition)12 StreamDefinition (org.apache.heron.eco.definition.StreamDefinition)12 Test (org.junit.Test)10 ArrayList (java.util.ArrayList)9 EcoTopologyDefinition (org.apache.heron.eco.definition.EcoTopologyDefinition)3 HashMap (java.util.HashMap)2 Fields (org.apache.heron.api.tuple.Fields)2 ComponentStream (org.apache.heron.eco.definition.ComponentStream)2 Fields (org.apache.storm.tuple.Fields)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 FileInputStream (java.io.FileInputStream)1 InputStream (java.io.InputStream)1 IBasicBolt (org.apache.heron.api.bolt.IBasicBolt)1 IRichBolt (org.apache.heron.api.bolt.IRichBolt)1 IStatefulWindowedBolt (org.apache.heron.api.bolt.IStatefulWindowedBolt)1 IWindowedBolt (org.apache.heron.api.bolt.IWindowedBolt)1 BoltDeclarer (org.apache.heron.api.topology.BoltDeclarer)1 BoltDefinition (org.apache.heron.eco.definition.BoltDefinition)1 BoltDeclarer (org.apache.storm.topology.BoltDeclarer)1 IBasicBolt (org.apache.storm.topology.IBasicBolt)1