Examples with Streamlet com.twitter.heron.streamlet.Streamlet used on opensource projects

Example 1 with Streamlet

use of com.twitter.heron.streamlet.Streamlet in project incubator-heron by apache.

the class StreamletImplTest method testCalculatedDefaultStageNames.

@Test
@SuppressWarnings("unchecked")
public void testCalculatedDefaultStageNames() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = StreamletImpl.createSupplierStreamlet(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertEquals(supplierStreamlet.getChildren().size(), 0);
    // apply the consumer function
    baseStreamlet.consume((SerializableConsumer<String>) s -> {
    });
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder builder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(builder, stageNames);
    // verify SupplierStreamlet
    assertTrue(supplierStreamlet.allBuilt());
    assertEquals(1, supplierStreamlet.getChildren().size());
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof ConsumerStreamlet);
    assertEquals("consumer1", supplierStreamlet.getChildren().get(0).getName());
    // verify stageNames
    assertEquals(2, stageNames.size());
    List<String> expectedStageNames = Arrays.asList("consumer1", "supplier1");
    assertTrue(stageNames.containsAll(expectedStageNames));
    // verify ConsumerStreamlet
    ConsumerStreamlet<String> consumerStreamlet = (ConsumerStreamlet<String>) supplierStreamlet.getChildren().get(0);
    assertEquals(0, consumerStreamlet.getChildren().size());
}

Example 2 with Streamlet

use of com.twitter.heron.streamlet.Streamlet in project incubator-heron by apache.

the class StreamletImplTest method testSimpleBuild.

@Test
@SuppressWarnings("unchecked")
public void testSimpleBuild() throws Exception {
    Streamlet<String> baseStreamlet = StreamletImpl.createSupplierStreamlet(() -> "sa re ga ma");
    baseStreamlet.flatMap(x -> Arrays.asList(x.split(" "))).reduceByKeyAndWindow(x -> x, x -> 1, WindowConfig.TumblingCountWindow(10), (x, y) -> x + y);
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder builder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(builder, stageNames);
    assertTrue(supplierStreamlet.allBuilt());
    assertEquals(supplierStreamlet.getChildren().size(), 1);
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof FlatMapStreamlet);
    FlatMapStreamlet<String, String> fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof ReduceByKeyAndWindowStreamlet);
    ReduceByKeyAndWindowStreamlet<String, Integer, Integer> rStreamlet = (ReduceByKeyAndWindowStreamlet<String, Integer, Integer>) fStreamlet.getChildren().get(0);
    assertEquals(rStreamlet.getChildren().size(), 0);
}

Example 3 with Streamlet

use of com.twitter.heron.streamlet.Streamlet in project incubator-heron by apache.

the class StreamletImplTest method testStreamletNameIfDuplicateNameIsSet.

@Test(expected = RuntimeException.class)
public void testStreamletNameIfDuplicateNameIsSet() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = StreamletImpl.createSupplierStreamlet(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    // set duplicate streamlet name and expect thrown exception
    supplierStreamlet.map((content) -> content.toUpperCase()).setName("MyMapStreamlet").map((content) -> content + "_test_suffix").setName("MyMapStreamlet");
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder builder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(builder, stageNames);
}

Example 4 with Streamlet

use of com.twitter.heron.streamlet.Streamlet in project incubator-heron by apache.

the class ImpressionsAndClicksTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    // A KVStreamlet is produced. Each element is a KeyValue object where the key
    // is the impression ID and the user ID is the value.
    Streamlet<AdImpression> impressions = processingGraphBuilder.newSource(AdImpression::new);
    // A KVStreamlet is produced. Each element is a KeyValue object where the key
    // is the ad ID and the user ID is the value.
    Streamlet<AdClick> clicks = processingGraphBuilder.newSource(AdClick::new);
    /**
     * Here, the impressions KVStreamlet is joined to the clicks KVStreamlet.
     */
    impressions.join(// The other streamlet that's being joined to
    clicks, // Key extractor for the impressions streamlet
    impression -> impression.getUserId(), // Key extractor for the clicks streamlet
    click -> click.getUserId(), // Window configuration for the join operation
    WindowConfig.TumblingCountWindow(25), // Join type (inner join means that all elements from both streams will be included)
    JoinType.INNER, // if the ad IDs match between the elements (or a value of 0 if they don't).
    (user1, user2) -> (user1.getAdId().equals(user2.getAdId())) ? 1 : 0).reduceByKeyAndWindow(// Key extractor for the reduce operation
    kv -> String.format("user-%s", kv.getKey().getKey()), // Value extractor for the reduce operation
    kv -> kv.getValue(), // Window configuration for the reduce operation
    WindowConfig.TumblingCountWindow(50), // A running cumulative total is calculated for each key
    (cumulative, incoming) -> cumulative + incoming).consume(kw -> {
        LOG.info(String.format("(user: %s, clicks: %d)", kw.getKey().getKey(), kw.getValue()));
    });
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}

Example 5 with Streamlet

use of com.twitter.heron.streamlet.Streamlet in project incubator-heron by apache.

the class StreamletCloneTopology method main.

/**
 * All Heron topologies require a main function that defines the topology's behavior
 * at runtime
 */
public static void main(String[] args) throws Exception {
    Builder processingGraphBuilder = Builder.newBuilder();
    /**
     * A supplier streamlet of random GameScore objects is cloned into two
     * separate streamlets.
     */
    List<Streamlet<GameScore>> splitGameScoreStreamlet = processingGraphBuilder.newSource(GameScore::new).clone(2);
    /**
     * Elements in the first cloned streamlet go to the database sink.
     */
    splitGameScoreStreamlet.get(0).toSink(new DatabaseSink());
    /**
     * Elements in the second cloned streamlet go to the logging sink.
     */
    splitGameScoreStreamlet.get(1).toSink(new FormattedLogSink());
    Config config = Config.defaultConfig();
    // Fetches the topology name from the first command-line argument
    String topologyName = StreamletUtils.getTopologyName(args);
    // Finally, the processing graph and configuration are passed to the Runner, which converts
    // the graph into a Heron topology that can be run in a Heron cluster.
    new Runner().run(topologyName, config, processingGraphBuilder);
}