use of org.apache.storm.generated.Bolt in project storm by apache.
the class Testing method captureTopology.
/**
* Rewrites a topology so that all the tuples flowing through it are captured
* @param topology the topology to rewrite
* @return the modified topology and a new Bolt that can retrieve the
* captured tuples.
*/
public static CapturedTopology<StormTopology> captureTopology(StormTopology topology) {
//Don't modify the original
topology = topology.deepCopy();
TupleCaptureBolt capturer = new TupleCaptureBolt();
Map<GlobalStreamId, Grouping> captureBoltInputs = new HashMap<>();
for (Map.Entry<String, SpoutSpec> spoutEntry : topology.get_spouts().entrySet()) {
String id = spoutEntry.getKey();
for (Entry<String, StreamInfo> streamEntry : spoutEntry.getValue().get_common().get_streams().entrySet()) {
String stream = streamEntry.getKey();
StreamInfo info = streamEntry.getValue();
if (info.is_direct()) {
captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
} else {
captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
}
}
}
for (Entry<String, Bolt> boltEntry : topology.get_bolts().entrySet()) {
String id = boltEntry.getKey();
for (Entry<String, StreamInfo> streamEntry : boltEntry.getValue().get_common().get_streams().entrySet()) {
String stream = streamEntry.getKey();
StreamInfo info = streamEntry.getValue();
if (info.is_direct()) {
captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareDirectGrouping());
} else {
captureBoltInputs.put(new GlobalStreamId(id, stream), Thrift.prepareGlobalGrouping());
}
}
}
topology.put_to_bolts(Utils.uuid(), new Bolt(Thrift.serializeComponentObject(capturer), Thrift.prepareComponentCommon(captureBoltInputs, new HashMap<>(), null)));
return new CapturedTopology<>(topology, capturer);
}
use of org.apache.storm.generated.Bolt in project storm by apache.
the class Task method mkTaskObject.
private Object mkTaskObject() {
StormTopology topology = systemTopologyContext.getRawTopology();
Map<String, SpoutSpec> spouts = topology.get_spouts();
Map<String, Bolt> bolts = topology.get_bolts();
Map<String, StateSpoutSpec> stateSpouts = topology.get_state_spouts();
Object result;
ComponentObject componentObject;
if (spouts.containsKey(componentId)) {
componentObject = spouts.get(componentId).get_spout_object();
} else if (bolts.containsKey(componentId)) {
componentObject = bolts.get(componentId).get_bolt_object();
} else if (stateSpouts.containsKey(componentId)) {
componentObject = stateSpouts.get(componentId).get_state_spout_object();
} else {
throw new RuntimeException("Could not find " + componentId + " in " + topology);
}
result = Utils.getSetComponentObject(componentObject);
if (result instanceof ShellComponent) {
if (spouts.containsKey(componentId)) {
result = new ShellSpout((ShellComponent) result);
} else {
result = new ShellBolt((ShellComponent) result);
}
}
if (result instanceof JavaObject) {
result = Thrift.instantiateJavaObject((JavaObject) result);
}
return result;
}
use of org.apache.storm.generated.Bolt in project storm by apache.
the class StormCommon method addAcker.
@SuppressWarnings("unchecked")
public static void addAcker(Map conf, StormTopology topology) {
int ackerNum = Utils.getInt(conf.get(Config.TOPOLOGY_ACKER_EXECUTORS), Utils.getInt(conf.get(Config.TOPOLOGY_WORKERS)));
Map<GlobalStreamId, Grouping> inputs = ackerInputs(topology);
Map<String, StreamInfo> outputStreams = new HashMap<String, StreamInfo>();
outputStreams.put(Acker.ACKER_ACK_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
outputStreams.put(Acker.ACKER_FAIL_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
outputStreams.put(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.directOutputFields(Arrays.asList("id", "time-delta-ms")));
Map<String, Object> ackerConf = new HashMap<>();
ackerConf.put(Config.TOPOLOGY_TASKS, ackerNum);
ackerConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
Bolt acker = Thrift.prepareSerializedBoltDetails(inputs, makeAckerBolt(), outputStreams, ackerNum, ackerConf);
for (Bolt bolt : topology.get_bolts().values()) {
ComponentCommon common = bolt.get_common();
common.put_to_streams(Acker.ACKER_ACK_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "ack-val")));
common.put_to_streams(Acker.ACKER_FAIL_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
common.put_to_streams(Acker.ACKER_RESET_TIMEOUT_STREAM_ID, Thrift.outputFields(Arrays.asList("id")));
}
for (SpoutSpec spout : topology.get_spouts().values()) {
ComponentCommon common = spout.get_common();
Map spoutConf = componentConf(spout);
spoutConf.put(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Utils.getInt(conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)));
common.set_json_conf(JSONValue.toJSONString(spoutConf));
common.put_to_streams(Acker.ACKER_INIT_STREAM_ID, Thrift.outputFields(Arrays.asList("id", "init-val", "spout-task")));
common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_ACK_STREAM_ID), Thrift.prepareDirectGrouping());
common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_FAIL_STREAM_ID), Thrift.prepareDirectGrouping());
common.put_to_inputs(Utils.getGlobalStreamId(Acker.ACKER_COMPONENT_ID, Acker.ACKER_RESET_TIMEOUT_STREAM_ID), Thrift.prepareDirectGrouping());
}
topology.put_to_bolts(Acker.ACKER_COMPONENT_ID, acker);
}
use of org.apache.storm.generated.Bolt in project storm by apache.
the class StreamBuilderTest method testRepartition.
@Test
public void testRepartition() throws Exception {
Stream<String> stream = streamBuilder.newStream(newSpout(Utils.DEFAULT_STREAM_ID), new ValueMapper<>(0));
stream.repartition(3).filter(x -> true).repartition(2).filter(x -> true).aggregate(new Count<>());
StormTopology topology = streamBuilder.build();
assertEquals(1, topology.get_spouts_size());
SpoutSpec spout = topology.get_spouts().get("spout1");
assertEquals(4, topology.get_bolts_size());
Bolt bolt1 = topology.get_bolts().get("bolt1");
Bolt bolt2 = topology.get_bolts().get("bolt2");
Bolt bolt3 = topology.get_bolts().get("bolt3");
Bolt bolt4 = topology.get_bolts().get("bolt4");
assertEquals(1, spout.get_common().get_parallelism_hint());
assertEquals(1, bolt1.get_common().get_parallelism_hint());
assertEquals(3, bolt2.get_common().get_parallelism_hint());
assertEquals(2, bolt3.get_common().get_parallelism_hint());
assertEquals(2, bolt4.get_common().get_parallelism_hint());
}
use of org.apache.storm.generated.Bolt in project incubator-atlas by apache.
the class StormAtlasHook method addBolts.
private void addBolts(Map<String, Bolt> bolts, Map<String, Referenceable> nodeEntities) throws IllegalAccessException {
for (Map.Entry<String, Bolt> entry : bolts.entrySet()) {
Referenceable boltInstance = createBoltInstance(entry.getKey(), entry.getValue());
nodeEntities.put(entry.getKey(), boltInstance);
}
}
Aggregations