Examples with TestTopologyBuilder org.apache.heron.integration_test.core.TestTopologyBuilder used on opensource projects

Example 1 with TestTopologyBuilder

use of org.apache.heron.integration_test.core.TestTopologyBuilder in project heron by twitter.

the class StreamletWithMapAndFlatMapAndFilterAndClone method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<String> streamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> SUMMER_MONTHS.contains(month.toLowerCase()) && incomingMonths.add(month.toLowerCase())).setName("summer-months").map((String word) -> word.substring(0, 3)).setName("summer-months-with-short-name");
    List<Streamlet<String>> clonedStreamlet = streamlet.clone(2);
    // Returns Summer Months with year
    clonedStreamlet.get(0).map((String month) -> month + "_2018");
    // Returns Summer Months with Uppercase
    clonedStreamlet.get(1).map((String month) -> month.toUpperCase());
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}

Example 2 with TestTopologyBuilder

use of org.apache.heron.integration_test.core.TestTopologyBuilder in project heron by twitter.

the class StreamletWithKeybyCountAndReduce method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<String> monthStreamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> incomingMonths.add(month.toLowerCase())).setName("unique-months");
    SerializableFunction<String, String> getSeason = month -> {
        if (SPRING_MONTHS.contains(month)) {
            return "spring";
        } else if (SUMMER_MONTHS.contains(month)) {
            return "summer";
        } else if (FALL_MONTHS.contains(month)) {
            return "fall";
        } else if (WINTER_MONTHS.contains(month)) {
            return "winter";
        } else {
            return "really?";
        }
    };
    SerializableFunction<String, Integer> getNumberOfDays = month -> {
        switch(month) {
            case "january":
                return 31;
            case "february":
                // Dont use this code in real projects
                return 28;
            case "march":
                return 31;
            case "april":
                return 30;
            case "may":
                return 31;
            case "june":
                return 30;
            case "july":
                return 31;
            case "august":
                return 31;
            case "september":
                return 30;
            case "october":
                return 31;
            case "november":
                return 30;
            case "december":
                return 31;
            default:
                // Shouldn't be here
                return -1;
        }
    };
    // Count months per season
    monthStreamlet.keyBy(getSeason, getNumberOfDays).setName("key-by-season").countByKey(x -> x.getKey()).setName("key-by-and-count").map(x -> String.format("%s: %d months", x.getKey(), x.getValue())).setName("to-string");
    // Sum days per season
    monthStreamlet.<String, Integer>reduceByKey(getSeason, getNumberOfDays, StreamletReducers::sum).setName("sum-by-season").map(x -> String.format("%s: %d days", x.getKey(), x.getValue())).setName("to-string-2");
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}

Example 3 with TestTopologyBuilder

use of org.apache.heron.integration_test.core.TestTopologyBuilder in project heron by twitter.

the class LocalReadWriteTopology method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3 || args.length > 4) {
        throw new RuntimeException("Expects 3 or 4 arguments, topology name, " + "inputFile, outputFile and max emit count (optional)");
    }
    String topologyName = args[0];
    String inputFile = args[1];
    String outputFile = args[2];
    TestTopologyBuilder builder = new TestTopologyBuilder(outputFile);
    builder.setTerminalBoltClass(LOCAL_AGGREGATOR_BOLT_CLASS);
    if (args.length == 3) {
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1);
    } else {
        int maxEmits = Integer.parseInt(args[3]);
        builder.setSpout("paused-local-spout", new PausedLocalFileSpout(inputFile), 1, maxEmits);
    }
    builder.setBolt("identity-bolt", new IdentityBolt(new Fields("line")), 1).shuffleGrouping("paused-local-spout");
    Config conf = new BasicConfig();
    HeronSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example 4 with TestTopologyBuilder

use of org.apache.heron.integration_test.core.TestTopologyBuilder in project heron by twitter.

the class AbstractTestTopology method submit.

public final void submit(Config userConf) throws AlreadyAliveException, InvalidTopologyException {
    TestTopologyBuilder builder = new TestTopologyBuilder(httpServerResultsUrl, httpServerStateUrl, stateUpdateToken, spoutWrapperType);
    Config conf = buildConfig(new BasicConfig());
    if (userConf != null) {
        conf.putAll(userConf);
    }
    HeronSubmitter.submitTopology(topologyName, conf, buildTopology(builder).createTopology());
}

Example 5 with TestTopologyBuilder

use of org.apache.heron.integration_test.core.TestTopologyBuilder in project heron by twitter.

the class StreamletWithSplitAndWithStream method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Map<String, SerializablePredicate<Integer>> splitter = new HashMap();
    splitter.put("all", i -> true);
    splitter.put("positive", i -> i > 0);
    splitter.put("negative", i -> i < 0);
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<Integer> multi = streamletBuilder.newSource(() -> atomicInteger.getAndIncrement()).setName("incremented-numbers-from--3").filter(i -> i <= 4).setName("numbers-lower-than-4").split(splitter).setName("split");
    multi.withStream("all").map((Integer i) -> String.format("all_%d", i));
    multi.withStream("positive").map((Integer i) -> String.format("pos_%d", i));
    multi.withStream("negative").map((Integer i) -> String.format("neg_%d", i));
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}