Examples with SerializableFunction org.apache.heron.streamlet.SerializableFunction used on opensource projects

Example 1 with SerializableFunction

use of org.apache.heron.streamlet.SerializableFunction in project heron by twitter.

the class StreamletWithKeybyCountAndReduce method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<String> monthStreamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> incomingMonths.add(month.toLowerCase())).setName("unique-months");
    SerializableFunction<String, String> getSeason = month -> {
        if (SPRING_MONTHS.contains(month)) {
            return "spring";
        } else if (SUMMER_MONTHS.contains(month)) {
            return "summer";
        } else if (FALL_MONTHS.contains(month)) {
            return "fall";
        } else if (WINTER_MONTHS.contains(month)) {
            return "winter";
        } else {
            return "really?";
        }
    };
    SerializableFunction<String, Integer> getNumberOfDays = month -> {
        switch(month) {
            case "january":
                return 31;
            case "february":
                // Dont use this code in real projects
                return 28;
            case "march":
                return 31;
            case "april":
                return 30;
            case "may":
                return 31;
            case "june":
                return 30;
            case "july":
                return 31;
            case "august":
                return 31;
            case "september":
                return 30;
            case "october":
                return 31;
            case "november":
                return 30;
            case "december":
                return 31;
            default:
                // Shouldn't be here
                return -1;
        }
    };
    // Count months per season
    monthStreamlet.keyBy(getSeason, getNumberOfDays).setName("key-by-season").countByKey(x -> x.getKey()).setName("key-by-and-count").map(x -> String.format("%s: %d months", x.getKey(), x.getValue())).setName("to-string");
    // Sum days per season
    monthStreamlet.<String, Integer>reduceByKey(getSeason, getNumberOfDays, StreamletReducers::sum).setName("sum-by-season").map(x -> String.format("%s: %d days", x.getKey(), x.getValue())).setName("to-string-2");
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}

Example 2 with SerializableFunction

use of org.apache.heron.streamlet.SerializableFunction in project heron by twitter.

the class JoinOperatorTest method getJoinOperator.

@SuppressWarnings({ "rawtypes", "unchecked" })
private JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> getJoinOperator(JoinType type) {
    SerializableFunction<KeyValue<String, String>, String> f = x -> x == null ? "null" : x.getKey();
    JoinOperator<String, KeyValue<String, String>, KeyValue<String, String>, String> joinOperator = new JoinOperator(type, "leftComponent", "rightComponent", f, f, (SerializableBiFunction<KeyValue<String, String>, KeyValue<String, String>, String>) (o, o2) -> (o == null ? "null" : o.getValue()) + (o2 == null ? "null" : o2.getValue()));
    joinOperator.prepare(new Config(), PowerMockito.mock(TopologyContext.class), new OutputCollector(new IOutputCollector() {

        @Override
        public void reportError(Throwable error) {
        }

        @Override
        public List<Integer> emit(String streamId, Collection<Tuple> anchors, List<Object> tuple) {
            emittedTuples.addAll(tuple);
            return null;
        }

        @Override
        public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple) {
        }

        @Override
        public void ack(Tuple input) {
        }

        @Override
        public void fail(Tuple input) {
        }
    }));
    return joinOperator;
}