use of org.apache.heron.eco.definition.StreamDefinition in project heron by twitter.
the class StreamBuilder method buildStreams.
protected void buildStreams(EcoExecutionContext executionContext, TopologyBuilder builder, ObjectBuilder objectBuilder) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchFieldException, InvocationTargetException {
EcoTopologyDefinition topologyDefinition = executionContext.getTopologyDefinition();
Map<String, ComponentStream> componentStreams = new HashMap<>();
HashMap<String, BoltDeclarer> declarers = new HashMap<>();
for (StreamDefinition stream : topologyDefinition.getStreams()) {
Object boltObj = executionContext.getBolt(stream.getTo());
BoltDeclarer declarer = declarers.get(stream.getTo());
if (boltObj instanceof IRichBolt) {
if (declarer == null) {
declarer = builder.setBolt(stream.getTo(), (IRichBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
declarers.put(stream.getTo(), declarer);
}
} else if (boltObj instanceof IBasicBolt) {
if (declarer == null) {
declarer = builder.setBolt(stream.getTo(), (IBasicBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
declarers.put(stream.getTo(), declarer);
}
} else if (boltObj instanceof IWindowedBolt) {
if (declarer == null) {
declarer = builder.setBolt(stream.getTo(), (IWindowedBolt) boltObj, topologyDefinition.parallelismForBolt(stream.getTo()));
declarers.put(stream.getTo(), declarer);
}
} else {
throw new IllegalArgumentException("Class does not appear to be a bolt: " + boltObj.getClass().getName());
}
GroupingDefinition grouping = stream.getGrouping();
// if the streamId is defined, use it for the grouping,
// otherwise assume default stream
String streamId = grouping.getStreamId() == null ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId();
switch(grouping.getType()) {
case SHUFFLE:
declarer.shuffleGrouping(stream.getFrom(), streamId);
break;
case FIELDS:
List<String> groupingArgs = grouping.getArgs();
if (groupingArgs == null) {
throw new IllegalArgumentException("You must supply arguments for Fields grouping");
}
declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(groupingArgs));
break;
case ALL:
declarer.allGrouping(stream.getFrom(), streamId);
break;
case GLOBAL:
declarer.globalGrouping(stream.getFrom(), streamId);
break;
case NONE:
declarer.noneGrouping(stream.getFrom(), streamId);
break;
case CUSTOM:
declarer.customGrouping(stream.getFrom(), streamId, buildCustomStreamGrouping(stream.getGrouping().getCustomClass(), executionContext, objectBuilder));
break;
default:
throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
}
}
executionContext.setStreams(componentStreams);
}
use of org.apache.heron.eco.definition.StreamDefinition in project heron by twitter.
the class StormStreamBuilderTest method buildStreams_SpoutToIWindowedBolt_CustomGrouping.
@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIWindowedBolt_CustomGrouping() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
final int iRichBoltParallelism = 1;
final String to = "to";
final String from = "from";
final String streamId = "id";
StreamDefinition streamDefinition = new StreamDefinition();
streamDefinition.setFrom(from);
streamDefinition.setTo(to);
streamDefinition.setId(streamId);
List<StreamDefinition> streams = new ArrayList<>();
streams.add(streamDefinition);
GroupingDefinition groupingDefinition = new GroupingDefinition();
groupingDefinition.setType(GroupingDefinition.Type.CUSTOM);
MockCustomObjectDefinition mockCustomObjectDefinition = new MockCustomObjectDefinition();
groupingDefinition.setCustomClass(mockCustomObjectDefinition);
List<String> args = new ArrayList<>();
args.add("arg1");
groupingDefinition.setArgs(args);
groupingDefinition.setStreamId(streamId);
streamDefinition.setGrouping(groupingDefinition);
MockIWindowedBolt mockIWindowedBolt = new MockIWindowedBolt();
MockCustomStreamGrouping mockCustomStreamGrouping = new MockCustomStreamGrouping();
when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
when(mockContext.getBolt(eq(to))).thenReturn(mockIWindowedBolt);
when(mockDefinition.getStreams()).thenReturn(streams);
when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
when(mockTopologyBuilder.setBolt(eq(to), eq(mockIWindowedBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
when(mockObjectBuilder.buildObject(eq(mockCustomObjectDefinition), eq(mockContext))).thenReturn(mockCustomStreamGrouping);
subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
verify(mockContext).getTopologyDefinition();
verify(mockContext).getBolt(eq(to));
verify(mockDefinition).parallelismForBolt(eq(to));
verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIWindowedBolt), eq(iRichBoltParallelism));
verify(mockBoltDeclarer).customGrouping(eq(from), eq(streamId), eq(mockCustomStreamGrouping));
verify(mockContext).setStreams(anyMap());
verify(mockDefinition).getStreams();
verify(mockObjectBuilder).buildObject(same(mockCustomObjectDefinition), same(mockContext));
}
use of org.apache.heron.eco.definition.StreamDefinition in project heron by twitter.
the class StormStreamBuilderTest method buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs.
@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
final int iRichBoltParallelism = 1;
final String to = "to";
final String from = "from";
final String streamId = "id";
StreamDefinition streamDefinition = new StreamDefinition();
streamDefinition.setFrom(from);
streamDefinition.setTo(to);
streamDefinition.setId(streamId);
List<StreamDefinition> streams = new ArrayList<>();
streams.add(streamDefinition);
GroupingDefinition groupingDefinition = new GroupingDefinition();
groupingDefinition.setType(GroupingDefinition.Type.FIELDS);
List<String> args = new ArrayList<>();
args.add("arg1");
groupingDefinition.setArgs(args);
groupingDefinition.setStreamId(streamId);
streamDefinition.setGrouping(groupingDefinition);
MockIBasicBolt mockIBasicBolt = new MockIBasicBolt();
when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
when(mockContext.getBolt(eq(to))).thenReturn(mockIBasicBolt);
when(mockDefinition.getStreams()).thenReturn(streams);
when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
when(mockTopologyBuilder.setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
verify(mockContext).getTopologyDefinition();
verify(mockContext).getBolt(eq(to));
verify(mockDefinition).parallelismForBolt(eq(to));
verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism));
verify(mockBoltDeclarer).fieldsGrouping(eq(from), eq(streamId), any(Fields.class));
verify(mockContext).setStreams(anyMap());
verify(mockDefinition).getStreams();
}
use of org.apache.heron.eco.definition.StreamDefinition in project heron by twitter.
the class HeronStreamBuilderTest method buildStreams_SpoutToIRichBolt_ShuffleGrouping.
@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIRichBolt_ShuffleGrouping() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
final int iRichBoltParallelism = 1;
final String to = "to";
final String from = "from";
final String streamId = "id";
StreamDefinition streamDefinition = new StreamDefinition();
streamDefinition.setFrom(from);
streamDefinition.setTo(to);
streamDefinition.setId(streamId);
List<StreamDefinition> streams = new ArrayList<>();
streams.add(streamDefinition);
GroupingDefinition groupingDefinition = new GroupingDefinition();
groupingDefinition.setType(GroupingDefinition.Type.SHUFFLE);
groupingDefinition.setStreamId(streamId);
streamDefinition.setGrouping(groupingDefinition);
MockIRichBolt mockIRichBolt = new MockIRichBolt();
when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
when(mockContext.getBolt(eq(to))).thenReturn(mockIRichBolt);
when(mockDefinition.getStreams()).thenReturn(streams);
when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
when(mockTopologyBuilder.setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
verify(mockContext).getTopologyDefinition();
verify(mockContext).getBolt(eq(to));
verify(mockDefinition).parallelismForBolt(eq(to));
verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIRichBolt), eq(iRichBoltParallelism));
verify(mockBoltDeclarer).shuffleGrouping(eq(from), eq(streamId));
verify(mockContext).setStreams(anyMap());
verify(mockDefinition).getStreams();
}
use of org.apache.heron.eco.definition.StreamDefinition in project heron by twitter.
the class HeronStreamBuilderTest method buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs.
@Test
@SuppressWarnings("unchecked")
public void buildStreams_SpoutToIBasicBolt_FieldsGroupingWithArgs() throws ClassNotFoundException, InvocationTargetException, NoSuchFieldException, InstantiationException, IllegalAccessException {
final int iRichBoltParallelism = 1;
final String to = "to";
final String from = "from";
final String streamId = "id";
StreamDefinition streamDefinition = new StreamDefinition();
streamDefinition.setFrom(from);
streamDefinition.setTo(to);
streamDefinition.setId(streamId);
List<StreamDefinition> streams = new ArrayList<>();
streams.add(streamDefinition);
GroupingDefinition groupingDefinition = new GroupingDefinition();
groupingDefinition.setType(GroupingDefinition.Type.FIELDS);
List<String> args = new ArrayList<>();
args.add("arg1");
groupingDefinition.setArgs(args);
groupingDefinition.setStreamId(streamId);
streamDefinition.setGrouping(groupingDefinition);
MockIBasicBolt mockIBasicBolt = new MockIBasicBolt();
when(mockContext.getTopologyDefinition()).thenReturn(mockDefinition);
when(mockContext.getBolt(eq(to))).thenReturn(mockIBasicBolt);
when(mockDefinition.getStreams()).thenReturn(streams);
when(mockDefinition.parallelismForBolt(eq(to))).thenReturn(iRichBoltParallelism);
when(mockTopologyBuilder.setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism))).thenReturn(mockBoltDeclarer);
subject.buildStreams(mockContext, mockTopologyBuilder, mockObjectBuilder);
verify(mockContext).getTopologyDefinition();
verify(mockContext).getBolt(eq(to));
verify(mockDefinition).parallelismForBolt(eq(to));
verify(mockTopologyBuilder).setBolt(eq(to), eq(mockIBasicBolt), eq(iRichBoltParallelism));
verify(mockBoltDeclarer).fieldsGrouping(eq(from), eq(streamId), any(Fields.class));
verify(mockContext).setStreams(anyMap());
verify(mockDefinition).getStreams();
}
Aggregations