Examples with Config org.apache.heron.streamlet.Config used on opensource projects

Example 11 with Config

use of org.apache.heron.streamlet.Config in project heron by twitter.

the class StreamletCloneTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    /**
     * A supplier streamlet of random GameScore objects is cloned into two
     * separate streamlets.
     */
    List<Streamlet<GameScore>> splitGameScoreStreamlet = processingGraphBuilder.newSource(GameScore::new).clone(2);
    /**
     * Elements in the first cloned streamlet go to the database sink.
     */
    splitGameScoreStreamlet.get(0).toSink(new DatabaseSink()).setName("sink0");
    /**
     * Elements in the second cloned streamlet go to the logging sink.
     */
    splitGameScoreStreamlet.get(1).toSink(new FormattedLogSink()).setName("sink1");
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 12 with Config

use of org.apache.heron.streamlet.Config in project heron by twitter.

the class WindowedWordCountTopology method main.

public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    processingGraphBuilder.newSource(() -> StreamletUtils.randomFromList(SENTENCES)).setName("random-sentences-source").flatMap(sentence -> Arrays.asList(sentence.toLowerCase().split("\\s+"))).setName("flatten-into-individual-words").reduceByKeyAndWindow(// The key extractor (the word is left unchanged)
    word -> word, // Value extractor (the value is always 1)
    word -> 1, WindowConfig.TumblingCountWindow(50), StreamletReducers::sum).setName("reduce-operation").consume(kv -> {
        String logMessage = String.format("(word: %s, count: %d)", kv.getKey().getKey(), kv.getValue());
        LOG.info(logMessage);
    });
    // The topology's parallelism (the number of containers across which the topology's
    // processing instance will be split) can be defined via the second command-line
    // argument (or else the default of 2 will be used).
    int topologyParallelism = StreamletUtils.getParallelism(args, 2);
    Config config = Config.newBuilder().setNumContainers(topologyParallelism).build();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 13 with Config

use of org.apache.heron.streamlet.Config in project heron by twitter.

the class ComponentConfigTopology method main.

public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    processingGraphBuilder.newSource(() -> StreamletUtils.randomFromList(SENTENCES)).setName("random-sentences-source").flatMap(sentence -> Arrays.asList(sentence.toLowerCase().split("\\s+"))).setName("flatten-into-individual-words").consume(w -> {
        String logMessage = String.format("(word: %s)", w);
        LOG.info(logMessage);
    }).setName("consumer");
    // The topology's parallelism (the number of containers across which the topology's
    // processing instance will be split) can be defined via the second command-line
    // argument (or else the default of 2 will be used).
    int topologyParallelism = StreamletUtils.getParallelism(args, 2);
    Config config = Config.newBuilder().setNumContainers(topologyParallelism).setPerContainerCpu(1).build();
    config.getHeronConfig().setComponentCpu("random-sentences-source", 0.3);
    config.getHeronConfig().setComponentRam("random-sentences-source", ByteAmount.fromMegabytes(300));
    config.getHeronConfig().setComponentCpu("flatten-into-individual-words", 0.3);
    config.getHeronConfig().setComponentRam("flatten-into-individual-words", ByteAmount.fromMegabytes(300));
    config.getHeronConfig().setComponentCpu("consumer", 0.2);
    config.getHeronConfig().setComponentRam("consumer", ByteAmount.fromMegabytes(200));
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 14 with Config

use of org.apache.heron.streamlet.Config in project heron by twitter.

the class StreamletImplTest method testConfigBuilderDefaultConfig.

@Test
public void testConfigBuilderDefaultConfig() {
    Config defaultConfig = Config.defaultConfig();
    assertEquals(defaultConfig.getSerializer(), Config.Serializer.KRYO);
    assertEquals(0, Double.compare(defaultConfig.getPerContainerCpu(), -1.0));
    assertEquals(defaultConfig.getPerContainerRam(), ByteAmount.fromBytes(-1).asBytes());
    assertEquals(defaultConfig.getDeliverySemantics(), Config.DeliverySemantics.ATMOST_ONCE);
    org.apache.heron.api.Config conf = defaultConfig.getHeronConfig();
    assertFalse(conf.containsKey(org.apache.heron.api.Config.TOPOLOGY_CONTAINER_CPU_REQUESTED));
    assertFalse(conf.containsKey(org.apache.heron.api.Config.TOPOLOGY_CONTAINER_MAX_CPU_HINT));
    assertFalse(conf.containsKey(org.apache.heron.api.Config.TOPOLOGY_CONTAINER_RAM_REQUESTED));
    assertFalse(conf.containsKey(org.apache.heron.api.Config.TOPOLOGY_CONTAINER_MAX_RAM_HINT));
}