Examples with Tuple org.apache.heron.api.tuple.Tuple used on opensource projects

Example 16 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class GeneralReduceByKeyAndWindowOperatorTest method getTupleWindow.

private TupleWindow getTupleWindow(int nkeys, int count) {
    TopologyAPI.StreamId componentStreamId = TopologyAPI.StreamId.newBuilder().setComponentName("sourceComponent").setId("default").build();
    List<Tuple> tuples = new LinkedList<>();
    for (int i = 0; i < nkeys; i++) {
        for (int j = 0; j < count; ++j) {
            Tuple tuple = getTuple(componentStreamId, new Fields("a"), new Values(new KeyValue<>(String.valueOf(i), j)));
            tuples.add(tuple);
        }
    }
    TupleWindow tupleWindow = new TupleWindowImpl(tuples, new LinkedList<>(), new LinkedList<>(), startTime, endTime);
    return tupleWindow;
}

Example 17 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class JoinOperatorTest method getJoinOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> getJoinOperator(JoinType type) {
    SerializableFunction<KeyValue<String, String>, String> f = x -> x == null ? "null" : x.getKey();
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = new JoinOperator(type, "leftComponent", "rightComponent", f, f, (SerializableBiFunction<KeyValue<String, String>, KeyValue<String, String>, String>) (o, o2) -> (o == null ? "null" : o.getValue()) + (o2 == null ? "null" : o2.getValue()));
    joinOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return joinOperator;
}

Example 18 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class KeyByOperatorTest method getKeyByOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private KeyByOperator<String, String, Integer> getKeyByOperator() {
    KeyByOperator<String, String, Integer> keyByOperator = new KeyByOperator<String, String, Integer>(x -> (Integer.valueOf(x) % 2 == 0) ? "even" : "odd", x -> Integer.valueOf(x));
    keyByOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return keyByOperator;
}

Example 19 with Tuple

use of org.apache.heron.api.tuple.Tuple in project heron by twitter.

the class WindowedBoltExecutorTest method testExecuteWithLateTupleStream.

@Test
public void testExecuteWithLateTupleStream() throws Exception {
    testWindowedBolt = new TestWindowedBolt();
    testWindowedBolt.withTimestampField("ts");
    executor = new WindowedBoltExecutor(testWindowedBolt);
    TopologyContext context = getTopologyContext();
    Mockito.when(context.getThisStreams()).thenReturn(new HashSet<>(Arrays.asList("default", "$late")));
    OutputCollector outputCollector = Mockito.mock(OutputCollector.class);
    Map<String, Object> conf = new HashMap<>();
    conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 100000);
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WINDOW_LENGTH_DURATION_MS, 20L);
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_SLIDING_INTERVAL_DURATION_MS, 10L);
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_LATE_TUPLE_STREAM, "$late");
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_TUPLE_TIMESTAMP_MAX_LAG_MS, 5L);
    // Trigger manually to avoid timing issues
    conf.put(WindowingConfigs.TOPOLOGY_BOLTS_WATERMARK_EVENT_INTERVAL_MS, 1_000_000L);
    executor.prepare(conf, context, outputCollector);
    long[] timestamps = { 603, 605, 607, 618, 626, 636, 600 };
    List<Tuple> tuples = new ArrayList<>(timestamps.length);
    for (long ts : timestamps) {
        Tuple tuple = getTuple("s1", new Fields("ts"), new Values(ts));
        tuples.add(tuple);
        executor.execute(tuple);
        // Update the watermark to this timestamp
        executor.waterMarkEventGenerator.run();
    }
    System.out.println(testWindowedBolt.tupleWindows);
    Tuple tuple = tuples.get(tuples.size() - 1);
    Mockito.verify(outputCollector).emit("$late", Arrays.asList(tuple), new Values(tuple));
}