Examples with Tuple com.twitter.heron.api.tuple.Tuple used on opensource projects

Example 1 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project heron by twitter.

the class LaunchRunnerTest method createTopology.

public static TopologyAPI.Topology createTopology(com.twitter.heron.api.Config heronConfig) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new BaseRichSpout() {

        private static final long serialVersionUID = -762965195665496156L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    }, 2);
    builder.setBolt("bolt-1", new BaseBasicBolt() {

        private static final long serialVersionUID = -5738458486388778812L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    }, 1);
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(TOPOLOGY_NAME).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 2 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project heron by twitter.

the class BoltOutputCollectorImpl method admitBoltTuple.

/////////////////////////////////////////////////////////
// Following private methods are internal implementations
/////////////////////////////////////////////////////////
private List<Integer> admitBoltTuple(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
    // First check whether this tuple is sane
    helper.checkOutputSchema(streamId, tuple);
    // customGroupingTargetTaskIds will be null if this stream is not CustomStreamGrouping
    List<Integer> customGroupingTargetTaskIds = helper.chooseTasksForCustomStreamGrouping(streamId, tuple);
    // Invoke user-defined emit task hook
    helper.getTopologyContext().invokeHookEmit(tuple, streamId, customGroupingTargetTaskIds);
    // Start construct the data tuple
    HeronTuples.HeronDataTuple.Builder bldr = HeronTuples.HeronDataTuple.newBuilder();
    // set the key. This is mostly ignored
    bldr.setKey(0);
    if (customGroupingTargetTaskIds != null) {
        // It is a CustomStreamGrouping
        for (Integer taskId : customGroupingTargetTaskIds) {
            bldr.addDestTaskIds(taskId);
        }
    }
    // Set the anchors for a tuple
    if (anchors != null) {
        // This message is rooted
        Set<HeronTuples.RootId> mergedRoots = new HashSet<HeronTuples.RootId>();
        for (Tuple tpl : anchors) {
            if (tpl instanceof TupleImpl) {
                TupleImpl t = (TupleImpl) tpl;
                mergedRoots.addAll(t.getRoots());
            }
        }
        for (HeronTuples.RootId rt : mergedRoots) {
            bldr.addRoots(rt);
        }
    }
    long tupleSizeInBytes = 0;
    long startTime = System.nanoTime();
    // Serialize it
    for (Object obj : tuple) {
        byte[] b = serializer.serialize(obj);
        ByteString bstr = ByteString.copyFrom(b);
        bldr.addValues(bstr);
        tupleSizeInBytes += b.length;
    }
    long latency = System.nanoTime() - startTime;
    boltMetrics.serializeDataTuple(streamId, latency);
    // submit to outputter
    outputter.addDataTuple(streamId, bldr, tupleSizeInBytes);
    // Update metrics
    boltMetrics.emittedTuple(streamId);
    // TODO:- remove this after changing the api
    return null;
}

Example 3 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class LaunchRunnerTest method createTopology.

public static TopologyAPI.Topology createTopology(com.twitter.heron.api.Config heronConfig) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new BaseRichSpout() {

        private static final long serialVersionUID = -762965195665496156L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    }, 2);
    builder.setBolt("bolt-1", new BaseBasicBolt() {

        private static final long serialVersionUID = -5738458486388778812L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    }, 1);
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(TOPOLOGY_NAME).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 4 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class JoinOperatorTest method getJoinOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> getJoinOperator(JoinType type) {
    SerializableFunction<KeyValue<String, String>, String> f = x -> x == null ? "null" : x.getKey();
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = new JoinOperator(type, "leftComponent", "rightComponent", f, f, (SerializableBiFunction<KeyValue<String, String>, KeyValue<String, String>, String>) (o, o2) -> (o == null ? "null" : o.getValue()) + (o2 == null ? "null" : o2.getValue()));
    joinOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return joinOperator;
}

Example 5 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class JoinOperatorTest method getTupleWindow.

private TupleWindow getTupleWindow() {
    TopologyAPI.StreamId leftComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("leftComponent").setId("s1").build();
    TopologyAPI.StreamId rightComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("rightComponent").setId("s1").build();
    List<Tuple> tuples = new LinkedList<>();
    for (int i = 0; i < 5; i++) {
        Tuple tuple;
        if (i % 2 == 0) {
            tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
        } else {
            tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
        }
        tuples.add(tuple);
    }
    for (int i = 5; i < 8; i++) {
        Tuple tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key2", String.valueOf(i))));
        tuples.add(tuple);
    }
    for (int i = 8; i < 12; i++) {
        Tuple tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key3", String.valueOf(i))));
        tuples.add(tuple);
    }
    TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
    return tupleWindow;
}