Examples with Fields org.apache.heron.api.tuple.Fields used on opensource projects

Example 11 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class JoinOperatorTest method getTupleWindow.

private TupleWindow getTupleWindow() {
    TopologyAPI.StreamId leftComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("leftComponent").setId("s1").build();
    TopologyAPI.StreamId rightComponentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("rightComponent").setId("s1").build();
    List<Tuple> tuples = new LinkedList<>();
    for (int i = 0; i < 5; i++) {
        Tuple tuple;
        if (i % 2 == 0) {
            tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
        } else {
            tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key1", String.valueOf(i))));
        }
        tuples.add(tuple);
    }
    for (int i = 5; i < 8; i++) {
        Tuple tuple = getTuple(leftComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key2", String.valueOf(i))));
        tuples.add(tuple);
    }
    for (int i = 8; i < 12; i++) {
        Tuple tuple = getTuple(rightComponentStreamId, new Fields("a"), new Values(new KeyValue<String, String>("key3", String.valueOf(i))));
        tuples.add(tuple);
    }
    TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
    return tupleWindow;
}

Example 12 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class ReduceByKeyAndWindowOperatorTest method getTupleWindow.

private TupleWindow getTupleWindow(int nkeys, int count) {
    TopologyAPI.StreamId componentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("sourceComponent").setId("default").build();
    List<Tuple> tuples = new LinkedList<>();
    for (int i = 0; i < nkeys; i++) {
        for (int j = 0; j < count; ++j) {
            Tuple tuple = getTuple(componentStreamId, new Fields("a"), new Values(String.valueOf(i)));
            tuples.add(tuple);
        }
    }
    TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
    return tupleWindow;
}

Example 13 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class WindowedBoltExecutorTest method testExecuteWithTs.

@Test
public void testExecuteWithTs() throws Exception {
    long[] timestamps = { 603, 605, 607, 618, 626, 636 };
    for (long ts : timestamps) {
        executor.execute(getTuple("s1", new Fields("ts"), new Values(ts)));
    }
    // Thread.sleep(120);
    executor.waterMarkEventGenerator.run();
    // System.out.println(testWindowedBolt.tupleWindows);
    assertEquals(3, testWindowedBolt.tupleWindows.size());
    TupleWindow first = testWindowedBolt.tupleWindows.get(0);
    assertArrayEquals(new long[] { 603, 605, 607 }, new long[] { (long) first.get().get(0).getValue(0), (long) first.get().get(1).getValue(0), (long) first.get().get(2).getValue(0) });
    TupleWindow second = testWindowedBolt.tupleWindows.get(1);
    assertArrayEquals(new long[] { 603, 605, 607, 618 }, new long[] { (long) second.get().get(0).getValue(0), (long) second.get().get(1).getValue(0), (long) second.get().get(2).getValue(0), (long) second.get().get(3).getValue(0) });
    TupleWindow third = testWindowedBolt.tupleWindows.get(2);
    assertArrayEquals(new long[] { 618, 626 }, new long[] { (long) third.get().get(0).getValue(0), (long) third.get().get(1).getValue(0) });
}

Example 14 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class TopologyTests method createTopologyWithConnection.

/**
 * Create Topology proto object using HeronSubmitter API.
 *
 * @param heronConfig desired config params.
 * @param spouts spoutName -> parallelism
 * @param bolts boltName -> parallelism
 * @param connections connect default stream from value to key.
 * @return topology proto.
 */
public static TopologyAPI.Topology createTopologyWithConnection(String topologyName, Config heronConfig, Map<String, Integer> spouts, Map<String, Integer> bolts, Map<String, String> connections) {
    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout baseSpout = new BaseRichSpout() {

        private static final long serialVersionUID = -719523487475322625L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
            declarer.declare(new Fields("field1"));
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    };
    BaseBasicBolt basicBolt = new BaseBasicBolt() {

        private static final long serialVersionUID = 2544765902130713628L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    };
    for (String spout : spouts.keySet()) {
        builder.setSpout(spout, baseSpout, spouts.get(spout));
    }
    for (String bolt : bolts.keySet()) {
        BoltDeclarer boltDeclarer = builder.setBolt(bolt, basicBolt, bolts.get(bolt));
        if (connections.containsKey(bolt)) {
            boltDeclarer.shuffleGrouping(connections.get(bolt));
        }
    }
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(topologyName).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 15 with Fields

use of org.apache.heron.api.tuple.Fields in project heron by twitter.

the class StatefulWordCountTopology method main.

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length < 1) {
        throw new RuntimeException("Specify topology name");
    }
    int parallelism = 1;
    if (args.length > 1) {
        parallelism = Integer.parseInt(args[1]);
    }
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism).fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
    conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
    conf.setTopologyStatefulCheckpointIntervalSecs(20);
    // configure component resources
    conf.setComponentRam("word", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
    conf.setComponentRam("consumer", ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
    // configure container resources
    conf.setContainerDiskRequested(ExampleResources.getContainerDisk(2 * parallelism, parallelism));
    conf.setContainerRamRequested(ExampleResources.getContainerRam(4 * parallelism, parallelism));
    conf.setContainerCpuRequested(ExampleResources.getContainerCpu(2 * parallelism, parallelism));
    HeronSubmitter.submitTopology(args[0], conf, builder.createTopology());
}