Examples with Tuple com.twitter.heron.api.tuple.Tuple used on opensource projects

Example 11 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
   * Create Topology proto object using HeronSubmitter API.
   *
   * @param heronConfig desired config params.
   * @param spouts spoutName -> parallelism
   * @param bolts boltName -> parallelism
   * @param connections connect default stream from value to key.
   * @return topology proto.
   */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 12 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project heron by twitter.

the class PhysicalPlanUtilTest method getTestTopology.

/**
   * Construct the test topology
   */
public static TopologyAPI.Topology getTestTopology() {
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    topologyBuilder.setSpout("word", new BaseRichSpout() {

        private static final long serialVersionUID = 5406114907377311020L;

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
            outputFieldsDeclarer.declare(new Fields("word"));
        }

        @Override
        public void open(Map<String, Object> map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
        }

        @Override
        public void nextTuple() {
        }
    }, 2);
    topologyBuilder.setBolt("exclaim", new BaseBasicBolt() {

        private static final long serialVersionUID = 4398578755681473899L;

        @Override
        public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
        }

        @Override
        public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
        }
    }, 2).shuffleGrouping("word");
    Config conf = new Config();
    conf.setDebug(true);
    conf.setMaxSpoutPending(10);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, "-XX:+HeapDumpOnOutOfMemoryError");
    conf.setComponentRam("word", ByteAmount.fromMegabytes(500));
    conf.setComponentRam("exclaim", ByteAmount.fromGigabytes(1));
    conf.setMessageTimeoutSecs(1);
    return topologyBuilder.createTopology().setName("topology-name").setConfig(conf).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 13 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class WindowedBoltExecutorTest method testExecuteWithLateTupleStream.

@Test
public void testExecuteWithLateTupleStream() throws Exception {
    testWindowedBolt = new TestWindowedBolt();
    testWindowedBolt.withTimestampField("ts");
    executor = new WindowedBoltExecutor(testWindowedBolt);
    TopologyContext context = getTopologyContext();
    Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late")));
    OutputCollector outputCollector = Mockito.mock(OutputCollector.class);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000);
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20L);
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10L);
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late");
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5L);
    // Trigger manually to avoid timing issues
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 1_000_000L);
    executor.prepare(conf, context, outputCollector);
    long[] timestamps = { 603, 605, 607, 618, 626, 636, 600 };
    List<Tuple> tuples = new ArrayList<>(timestamps.length);
    for (long ts : timestamps) {
        Tuple tuple = getTuple("s1", new Fields("ts"), new Values(ts));
        tuples.add(tuple);
        executor.execute(tuple);
        // Update the watermark to this timestamp
        executor.waterMarkEventGenerator.run();
    }
    System.out.println(testWindowedBolt.tupleWindows);
    Tuple tuple = tuples.get(tuples.size() - 1);
    Mockito.verify(outputCollector).emit("$late", Arrays.asList(tuple), new Values(tuple));
}

Example 14 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class GeneralReduceByKeyAndWindowOperatorTest method getTupleWindow.

private TupleWindow getTupleWindow(int nkeys, int count) {
    TopologyAPI.StreamId componentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("sourceComponent").setId("default").build();
    List<Tuple> tuples = new LinkedList<>();
    for (int i = 0; i < nkeys; i++) {
        for (int j = 0; j < count; ++j) {
            Tuple tuple = getTuple(componentStreamId, new Fields("a"), new Values(new KeyValue<>(String.valueOf(i), j)));
            tuples.add(tuple);
        }
    }
    TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
    return tupleWindow;
}

Example 15 with Tuple

use of com.twitter.heron.api.tuple.Tuple in project incubator-heron by apache.

the class GeneralReduceByKeyAndWindowOperatorTest method getReduceByWindowOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private GeneralReduceByKeyAndWindowOperator<String, KeyValue<String, Integer>, Integer> getReduceByWindowOperator(Integer identity) {
    GeneralReduceByKeyAndWindowOperator<String, KeyValue<String, Integer>, Integer> reduceByWindowOperator = new GeneralReduceByKeyAndWindowOperator<>(x -> x.getKey(), identity, (o, o2) -> o + o2.getValue());
    reduceByWindowOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return reduceByWindowOperator;
}