Examples with SpoutOutputCollector org.apache.heron.api.spout.SpoutOutputCollector used on opensource projects

Example 1 with SpoutOutputCollector

use of org.apache.heron.api.spout.SpoutOutputCollector in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
 * Create Topology proto object using HeronSubmitter API.
 *
 * @param heronConfig desired config params.
 * @param spouts spoutName -> parallelism
 * @param bolts boltName -> parallelism
 * @param connections connect default stream from value to key.
 * @return topology proto.
 */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 2 with SpoutOutputCollector

use of org.apache.heron.api.spout.SpoutOutputCollector in project heron by twitter.

the class TopologyManagerTest method getTestTopology.

/**
 * Construct the test topology
 */
public static TopologyAPI.Topology getTestTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(STREAM_ID, new BaseRichSpout() {

        private static final long serialVersionUID = 5406114907377311020L;

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
            outputFieldsDeclarer.declare(new Fields(STREAM_ID));
        }

        @Override
        public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        }

        @Override
        public void nextTuple() {
        }
    }, 2);
    topologyBuilder.setBolt(BOLT_ID, new BaseBasicBolt() {

        private static final long serialVersionUID = 4398578755681473899L;

        @Override
        public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        }

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        }
    }, 2).shuffleGrouping(STREAM_ID);
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam(STREAM_ID, ByteAmount.fromMegabytes(500));
    conf.setComponentRam(BOLT_ID, ByteAmount.fromGigabytes(1));
    conf.setMessageTimeoutSecs(1);
    return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 3 with SpoutOutputCollector

use of org.apache.heron.api.spout.SpoutOutputCollector in project heron by twitter.

the class SpoutInstance method init.

@SuppressWarnings("unchecked")
@Override
public void init(State<Serializable, Serializable> state) {
    TopologyContextImpl topologyContext = helper.getTopologyContext();
    // Initialize the GlobalMetrics
    GlobalMetrics.init(topologyContext, systemConfig.getHeronMetricsExportInterval());
    spoutMetrics.registerMetrics(topologyContext);
    // Initialize the instanceState if the topology is stateful and spout is a stateful component
    if (isTopologyStateful && spout instanceof IStatefulComponent) {
        this.instanceState = state;
        ((IStatefulComponent<Serializable, Serializable>) spout).initState(instanceState);
        if (spillState) {
            if (FileUtils.isDirectoryExists(spillStateLocation)) {
                FileUtils.cleanDir(spillStateLocation);
            } else {
                FileUtils.createDirectory(spillStateLocation);
            }
        }
    }
    spout.open(topologyContext.getTopologyConfig(), topologyContext, new SpoutOutputCollector(collector));
    // Invoke user-defined prepare task hook
    topologyContext.invokeHookPrepare();
    // Init the CustomStreamGrouping
    helper.prepareForCustomStreamGrouping();
}

Example 4 with SpoutOutputCollector

use of org.apache.heron.api.spout.SpoutOutputCollector in project heron by twitter.

the class LaunchRunnerTest method createTopology.

public static TopologyAPI.Topology createTopology(org.apache.heron.api.Config heronConfig) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new BaseRichSpout() {

        private static final long serialVersionUID = -762965195665496156L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    }, 2);
    builder.setBolt("bolt-1", new BaseBasicBolt() {

        private static final long serialVersionUID = -5738458486388778812L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    }, 1);
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(TOPOLOGY_NAME).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 5 with SpoutOutputCollector

use of org.apache.heron.api.spout.SpoutOutputCollector in project heron by twitter.

the class IntegrationTestSpout method open.

@Override
public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector outputCollector) {
    // Here the spoutOutputCollector should be a default one
    // to emit tuples without adding MessageId
    this.spoutOutputCollector = outputCollector;
    delegateSpout.open(map, new TestTopologyContext(topologyContext), new SpoutOutputCollector(new IntegrationTestSpoutCollector(outputCollector)));
}