Examples with Streamlet org.apache.heron.streamlet.Streamlet used on opensource projects

Example 6 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class StreamletImplTest method testSimpleBuild.

@Test
@SuppressWarnings("unchecked")
public void testSimpleBuild() throws Exception {
    Streamlet<String> baseStreamlet = builder.newSource(() -> "sa re ga ma");
    baseStreamlet.flatMap(x -> Arrays.asList(x.split(" "))).reduceByKeyAndWindow(x -> x, x -> 1, WindowConfig.TumblingCountWindow(10), (x, y) -> x + y);
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
    assertTrue(supplierStreamlet.isFullyBuilt());
    assertEquals(supplierStreamlet.getChildren().size(), 1);
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof FlatMapStreamlet);
    FlatMapStreamlet<String, String> fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof ReduceByKeyAndWindowStreamlet);
    ReduceByKeyAndWindowStreamlet<String, Integer, Integer> rStreamlet = (ReduceByKeyAndWindowStreamlet<String, Integer, Integer>) fStreamlet.getChildren().get(0);
    assertEquals(rStreamlet.getChildren().size(), 0);
}

Example 7 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class ImpressionsAndClicksTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    // A KVStreamlet is produced. Each element is a KeyValue object where the key
    // is the impression ID and the user ID is the value.
    Streamlet<AdImpression> impressions = processingGraphBuilder.newSource(AdImpression::new);
    // A KVStreamlet is produced. Each element is a KeyValue object where the key
    // is the ad ID and the user ID is the value.
    Streamlet<AdClick> clicks = processingGraphBuilder.newSource(AdClick::new);
    /**
     * Here, the impressions KVStreamlet is joined to the clicks KVStreamlet.
     */
    impressions.join(// The other streamlet that's being joined to
    clicks, // Key extractor for the impressions streamlet
    impression -> impression.getUserId(), // Key extractor for the clicks streamlet
    click -> click.getUserId(), // Window configuration for the join operation
    WindowConfig.TumblingCountWindow(25), // Join type (inner join means that all elements from both streams will be included)
    JoinType.INNER, // if the ad IDs match between the elements (or a value of 0 if they don't).
    (user1, user2) -> (user1.getAdId().equals(user2.getAdId())) ? 1 : 0).reduceByKeyAndWindow(// Key extractor for the reduce operation
    kv -> String.format("user-%s", kv.getKey().getKey()), // Value extractor for the reduce operation
    kv -> kv.getValue(), // Window configuration for the reduce operation
    WindowConfig.TumblingCountWindow(50), // A running cumulative total is calculated for each key
    (cumulative, incoming) -> cumulative + incoming).consume(kw -> {
        LOG.info(String.format("(user: %s, clicks: %d)", kw.getKey().getKey(), kw.getValue()));
    });
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 8 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class StreamletCloneTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    /**
     * A supplier streamlet of random GameScore objects is cloned into two
     * separate streamlets.
     */
    List<Streamlet<GameScore>> splitGameScoreStreamlet = processingGraphBuilder.newSource(GameScore::new).clone(2);
    /**
     * Elements in the first cloned streamlet go to the database sink.
     */
    splitGameScoreStreamlet.get(0).toSink(new DatabaseSink()).setName("sink0");
    /**
     * Elements in the second cloned streamlet go to the logging sink.
     */
    splitGameScoreStreamlet.get(1).toSink(new FormattedLogSink()).setName("sink1");
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}