use of org.apache.heron.api.topology.TopologyContext in project heron by twitter.
the class WindowedBoltExecutorTest method testPrepareLateTupleStreamWithoutTs.
@Test
public void testPrepareLateTupleStreamWithoutTs() throws Exception {
Map<String, Object> conf = new HashMap<>();
conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20L);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10L);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late");
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5L);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 10L);
testWindowedBolt = new TestWindowedBolt();
executor = new WindowedBoltExecutor(testWindowedBolt);
TopologyContext context = getTopologyContext();
// emulate the call of withLateTupleStream method
Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late")));
try {
executor.prepare(conf, context, getOutputCollector());
fail();
} catch (IllegalArgumentException e) {
assertEquals(e.getMessage(), "Late tuple stream can be defined only when specifying a " + "timestamp field");
}
}
use of org.apache.heron.api.topology.TopologyContext in project heron by twitter.
the class WindowedBoltExecutorTest method testPrepareLateTupleStreamWithoutBuilder.
@Test
public void testPrepareLateTupleStreamWithoutBuilder() throws Exception {
Map<String, Object> conf = new HashMap<>();
conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20L);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10L);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late");
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5L);
conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 10L);
testWindowedBolt = new TestWindowedBolt();
testWindowedBolt.withTimestampField("ts");
executor = new WindowedBoltExecutor(testWindowedBolt);
TopologyContext context = getTopologyContext();
try {
executor.prepare(conf, context, getOutputCollector());
fail();
} catch (IllegalArgumentException e) {
assertEquals(e.getMessage(), "Stream for late tuples must be defined with the builder " + "method withLateTupleStream");
}
}
use of org.apache.heron.api.topology.TopologyContext in project heron by twitter.
the class ComplexSource method open.
@SuppressWarnings("rawtypes")
@Override
public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector outputCollector) {
super.open(map, topologyContext, outputCollector);
Context context = new ContextImpl(topologyContext, map, state);
generator.setup(context);
}
use of org.apache.heron.api.topology.TopologyContext in project heron by twitter.
the class TopologyTests method createTopologyWithConnection.
/**
* Create Topology proto object using HeronSubmitter API.
*
* @param heronConfig desired config params.
* @param spouts spoutName -> parallelism
* @param bolts boltName -> parallelism
* @param connections connect default stream from value to key.
* @return topology proto.
*/
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
TopologyBuilder builder = new TopologyBuilder();
BaseRichSpout baseSpout = new BaseRichSpout() {
private static final long serialVersionUID = -719523487475322625L;
public void declareOutputFields(OutputFieldsDeclarer declarer) {
declarer.declare(new Fields("field1"));
}
public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
}
public void nextTuple() {
}
};
BaseBasicBolt basicBolt = new BaseBasicBolt() {
private static final long serialVersionUID = 2544765902130713628L;
public void execute(Tuple input, BasicOutputCollector collector) {
}
public void declareOutputFields(OutputFieldsDeclarer declarer) {
}
};
for (String spout : spouts.keySet()) {
builder.setSpout(spout, baseSpout, spouts.get(spout));
}
for (String bolt : bolts.keySet()) {
BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
if (connections.containsKey(bolt)) {
boltDeclarer.shuffleGrouping(connections.get(bolt));
}
}
HeronTopology heronTopology = builder.createTopology();
return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}
use of org.apache.heron.api.topology.TopologyContext in project heron by twitter.
the class TopologyManagerTest method getTestTopology.
/**
* Construct the test topology
*/
public static TopologyAPI.Topology getTestTopology() {
TopologyBuilder topologyBuilder = new TopologyBuilder();
topologyBuilder.setSpout(STREAM_ID, new BaseRichSpout() {
private static final long serialVersionUID = 5406114907377311020L;
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
outputFieldsDeclarer.declare(new Fields(STREAM_ID));
}
@Override
public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
}
@Override
public void nextTuple() {
}
}, 2);
topologyBuilder.setBolt(BOLT_ID, new BaseBasicBolt() {
private static final long serialVersionUID = 4398578755681473899L;
@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
}
}, 2).shuffleGrouping(STREAM_ID);
Config conf = new Config();
conf.setDebug(true);
conf.setMaxSpoutPending(10);
conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
conf.setComponentRam(STREAM_ID, ByteAmount.fromMegabytes(500));
conf.setComponentRam(BOLT_ID, ByteAmount.fromGigabytes(1));
conf.setMessageTimeoutSecs(1);
return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}
Aggregations