Examples with Builder org.apache.heron.streamlet.Builder used on opensource projects

Example 11 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class IntegerProcessingTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder builder = Builder.newBuilder();
    Streamlet<Integer> zeroes = builder.newSource(() -> 0);
    builder.newSource(() -> ThreadLocalRandom.current().nextInt(1, 11)).setName("random-ints").map(i -> i + 1).setName("add-one").union(zeroes).setName("unify-streams").filter(i -> i != 2).setName("remove-twos").log();
    Config config = Config.newBuilder().setNumContainers(NUM_CONTAINERS).setPerContainerRamInGigabytes(GIGABYTES_OF_RAM).setPerContainerCpu(CPU).build();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, builder);
}

Example 12 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class StreamletCloneTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    /**
     * A supplier streamlet of random GameScore objects is cloned into two
     * separate streamlets.
     */
    List<Streamlet<GameScore>> splitGameScoreStreamlet = processingGraphBuilder.newSource(GameScore::new).clone(2);
    /**
     * Elements in the first cloned streamlet go to the database sink.
     */
    splitGameScoreStreamlet.get(0).toSink(new DatabaseSink()).setName("sink0");
    /**
     * Elements in the second cloned streamlet go to the logging sink.
     */
    splitGameScoreStreamlet.get(1).toSink(new FormattedLogSink()).setName("sink1");
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 13 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class WindowedWordCountTopology method main.

public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    processingGraphBuilder.newSource(() -> StreamletUtils.randomFromList(SENTENCES)).setName("random-sentences-source").flatMap(sentence -> Arrays.asList(sentence.toLowerCase().split("\\s+"))).setName("flatten-into-individual-words").reduceByKeyAndWindow(// The key extractor (the word is left unchanged)
    word -> word, // Value extractor (the value is always 1)
    word -> 1, WindowConfig.TumblingCountWindow(50), StreamletReducers::sum).setName("reduce-operation").consume(kv -> {
        String logMessage = String.format("(word: %s, count: %d)", kv.getKey().getKey(), kv.getValue());
        LOG.info(logMessage);
    });
    // The topology's parallelism (the number of containers across which the topology's
    // processing instance will be split) can be defined via the second command-line
    // argument (or else the default of 2 will be used).
    int topologyParallelism = StreamletUtils.getParallelism(args, 2);
    Config config = Config.newBuilder().setNumContainers(topologyParallelism).build();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 14 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class ComponentConfigTopology method main.

public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    processingGraphBuilder.newSource(() -> StreamletUtils.randomFromList(SENTENCES)).setName("random-sentences-source").flatMap(sentence -> Arrays.asList(sentence.toLowerCase().split("\\s+"))).setName("flatten-into-individual-words").consume(w -> {
        String logMessage = String.format("(word: %s)", w);
        LOG.info(logMessage);
    }).setName("consumer");
    // The topology's parallelism (the number of containers across which the topology's
    // processing instance will be split) can be defined via the second command-line
    // argument (or else the default of 2 will be used).
    int topologyParallelism = StreamletUtils.getParallelism(args, 2);
    Config config = Config.newBuilder().setNumContainers(topologyParallelism).setPerContainerCpu(1).build();
    config.getHeronConfig().setComponentCpu("random-sentences-source", 0.3);
    config.getHeronConfig().setComponentRam("random-sentences-source", ByteAmount.fromMegabytes(300));
    config.getHeronConfig().setComponentCpu("flatten-into-individual-words", 0.3);
    config.getHeronConfig().setComponentRam("flatten-into-individual-words", ByteAmount.fromMegabytes(300));
    config.getHeronConfig().setComponentCpu("consumer", 0.2);
    config.getHeronConfig().setComponentRam("consumer", ByteAmount.fromMegabytes(200));
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 15 with Builder

use of org.apache.heron.streamlet.Builder in project heron by twitter.

the class StreamletWithSplitAndWithStream method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Map<String, SerializablePredicate<Integer>> splitter = new HashMap();
    splitter.put("all", i -> true);
    splitter.put("positive", i -> i > 0);
    splitter.put("negative", i -> i < 0);
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<Integer> multi = streamletBuilder.newSource(() -> atomicInteger.getAndIncrement()).setName("incremented-numbers-from--3").filter(i -> i <= 4).setName("numbers-lower-than-4").split(splitter).setName("split");
    multi.withStream("all").map((Integer i) -> String.format("all_%d", i));
    multi.withStream("positive").map((Integer i) -> String.format("pos_%d", i));
    multi.withStream("negative").map((Integer i) -> String.format("neg_%d", i));
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}