Examples with Tuple org.apache.heron.api.tuple.Tuple used on opensource projects

Example 6 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class ReduceByKeyAndWindowOperator method execute.

@SuppressWarnings("unchecked")
@Override
public void execute(TupleWindow inputWindow) {
    Map<K, T> reduceMap = new HashMap<>();
    Map<K, Integer> windowCountMap = new HashMap<>();
    for (Tuple tuple : inputWindow.get()) {
        R tup = (R) tuple.getValue(0);
        addMap(reduceMap, windowCountMap, tup);
    }
    long startWindow;
    long endWindow;
    if (inputWindow.getStartTimestamp() == null) {
        startWindow = 0;
    } else {
        startWindow = inputWindow.getStartTimestamp();
    }
    if (inputWindow.getEndTimestamp() == null) {
        endWindow = 0;
    } else {
        endWindow = inputWindow.getEndTimestamp();
    }
    for (K key : reduceMap.keySet()) {
        Window window = new Window(startWindow, endWindow, windowCountMap.get(key));
        KeyedWindow<K> keyedWindow = new KeyedWindow<>(key, window);
        collector.emit(new Values(new KeyValue<>(keyedWindow, reduceMap.get(key))));
    }
}

Example 7 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
 * Create Topology proto object using HeronSubmitter API.
 *
 * @param heronConfig desired config params.
 * @param spouts spoutName -> parallelism
 * @param bolts boltName -> parallelism
 * @param connections connect default stream from value to key.
 * @return topology proto.
 */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 8 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class WindowedBoltExecutor method newWindowLifecycleListener.

protected WindowLifecycleListener<Tuple> newWindowLifecycleListener() {
    return new WindowLifecycleListener<Tuple>() {

        @Override
        public void onExpiry(List<Tuple> tuples) {
            for (Tuple tuple : tuples) {
                windowedOutputCollector.ack(tuple);
            }
        }

        @Override
        public void onActivation(List<Tuple> tuples, List<Tuple> newTuples, List<Tuple> expiredTuples, Long timestamp) {
            windowedOutputCollector.setContext(tuples);
            boltExecute(tuples, newTuples, expiredTuples, timestamp);
        }
    };
}

Example 9 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class TopologyManagerTest method getTestTopology.

/**
 * Construct the test topology
 */
public static TopologyAPI.Topology getTestTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout(STREAM_ID, new BaseRichSpout() {

        private static final long serialVersionUID = 5406114907377311020L;

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
            outputFieldsDeclarer.declare(new Fields(STREAM_ID));
        }

        @Override
        public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        }

        @Override
        public void nextTuple() {
        }
    }, 2);
    topologyBuilder.setBolt(BOLT_ID, new BaseBasicBolt() {

        private static final long serialVersionUID = 4398578755681473899L;

        @Override
        public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        }

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        }
    }, 2).shuffleGrouping(STREAM_ID);
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam(STREAM_ID, ByteAmount.fromMegabytes(500));
    conf.setComponentRam(BOLT_ID, ByteAmount.fromGigabytes(1));
    conf.setMessageTimeoutSecs(1);
    return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 10 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class BoltOutputCollectorImpl method admitBoltTuple.

// ///////////////////////////////////////////////////////
// Following private methods are internal implementations
// ///////////////////////////////////////////////////////
private List<Integer> admitBoltTuple(String streamId, Collection<Tuple> anchors, List<Object> tuple, Integer emitDirectTaskId) {
    if (getPhysicalPlanHelper().isTerminatedComponent()) {
        // No need to handle this tuple
        return null;
    }
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = initTupleBuilder(streamId, tuple, emitDirectTaskId);
    // Set the anchors for a tuple
    if (anchors != null) {
        // This message is rooted
        Set<HeronTuples.RootId> mergedRoots = new HashSet<>();
        for (Tuple tpl : anchors) {
            if (tpl instanceof TupleImpl) {
                TupleImpl t = (TupleImpl) tpl;
                mergedRoots.addAll(t.getRoots());
            }
        }
        for (HeronTuples.RootId rt : mergedRoots) {
            bldr.addRoots(rt);
        }
    }
    sendTuple(bldr, streamId, tuple);
    // TODO:- remove this after changing the API
    return null;
}