Examples with SupplierStreamlet org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet used on opensource projects

Example 1 with SupplierStreamlet

use of org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet in project heron by twitter.

the class StreamletImplTest method testStreamletNameIfDuplicateNameIsSet.

@Test(expected = RuntimeException.class)
public void testStreamletNameIfDuplicateNameIsSet() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    // set duplicate streamlet name and expect thrown exception
    supplierStreamlet.map((content) -> content.toUpperCase()).setName("MyMapStreamlet").map((content) -> content + "_test_suffix").setName("MyMapStreamlet");
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
}

Example 2 with SupplierStreamlet

use of org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet in project heron by twitter.

the class StreamletImplTest method testCustomStreamletWithGrouperFromBolt.

@Test
@SuppressWarnings("unchecked")
public void testCustomStreamletWithGrouperFromBolt() throws Exception {
    Streamlet<Double> baseStreamlet = builder.newSource(() -> Math.random());
    Streamlet<Double> streamlet = baseStreamlet.setNumPartitions(20).applyOperator(new MyBoltOperator(), new ShuffleStreamGrouping());
    assertTrue(streamlet instanceof CustomStreamlet);
    CustomStreamlet<Double, Double> mStreamlet = (CustomStreamlet<Double, Double>) streamlet;
    assertEquals(20, mStreamlet.getNumPartitions());
    SupplierStreamlet<Double> supplierStreamlet = (SupplierStreamlet<Double>) baseStreamlet;
    assertEquals(supplierStreamlet.getChildren().size(), 1);
    assertEquals(supplierStreamlet.getChildren().get(0), streamlet);
}

Example 3 with SupplierStreamlet

use of org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet in project heron by twitter.

the class StreamletImplTest method testCalculatedDefaultStageNames.

@Test
@SuppressWarnings("unchecked")
public void testCalculatedDefaultStageNames() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertEquals(supplierStreamlet.getChildren().size(), 0);
    // apply the consumer function
    baseStreamlet.consume((SerializableConsumer<String>) s -> {
    });
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
    // verify SupplierStreamlet
    assertTrue(supplierStreamlet.isFullyBuilt());
    assertEquals(1, supplierStreamlet.getChildren().size());
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof ConsumerStreamlet);
    assertEquals("consumer1", supplierStreamlet.getChildren().get(0).getName());
    // verify stageNames
    assertEquals(2, stageNames.size());
    List<String> expectedStageNames = Arrays.asList("consumer1", "supplier1");
    assertTrue(stageNames.containsAll(expectedStageNames));
    // verify ConsumerStreamlet
    ConsumerStreamlet<String> consumerStreamlet = (ConsumerStreamlet<String>) supplierStreamlet.getChildren().get(0);
    assertEquals(0, consumerStreamlet.getChildren().size());
}

Example 4 with SupplierStreamlet

use of org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet in project heron by twitter.

the class StreamletImplTest method testComplexBuild.

@Test
@SuppressWarnings("unchecked")
public void testComplexBuild() {
    // First source
    Streamlet<String> baseStreamlet1 = builder.newSource(() -> "sa re ga ma");
    Streamlet<String> leftStream = baseStreamlet1.flatMap(x -> Arrays.asList(x.split(" ")));
    // Second source
    Streamlet<String> baseStreamlet2 = builder.newSource(() -> "I Love You");
    Streamlet<String> rightStream = baseStreamlet2.flatMap(x -> Arrays.asList(x.split(" ")));
    // join
    leftStream.join(rightStream, x -> x, x -> x, WindowConfig.TumblingCountWindow(10), (x, y) -> x + y);
    SupplierStreamlet<String> supplierStreamlet1 = (SupplierStreamlet<String>) baseStreamlet1;
    SupplierStreamlet<String> supplierStreamlet2 = (SupplierStreamlet<String>) baseStreamlet2;
    assertFalse(supplierStreamlet1.isBuilt());
    assertFalse(supplierStreamlet2.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet1.build(topologyBuilder, stageNames);
    assertTrue(supplierStreamlet1.isBuilt());
    assertFalse(supplierStreamlet1.isFullyBuilt());
    supplierStreamlet2.build(topologyBuilder, stageNames);
    assertTrue(supplierStreamlet1.isFullyBuilt());
    assertTrue(supplierStreamlet2.isFullyBuilt());
    // go over all stuff
    assertEquals(supplierStreamlet1.getChildren().size(), 1);
    assertTrue(supplierStreamlet1.getChildren().get(0) instanceof FlatMapStreamlet);
    FlatMapStreamlet<String, String> fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet1.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof JoinStreamlet);
    JoinStreamlet<String, String, String, String> jStreamlet = (JoinStreamlet<String, String, String, String>) fStreamlet.getChildren().get(0);
    assertEquals(jStreamlet.getChildren().size(), 0);
    assertEquals(supplierStreamlet2.getChildren().size(), 1);
    assertTrue(supplierStreamlet2.getChildren().get(0) instanceof FlatMapStreamlet);
    fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet2.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof JoinStreamlet);
    jStreamlet = (JoinStreamlet<String, String, String, String>) fStreamlet.getChildren().get(0);
    assertEquals(jStreamlet.getChildren().size(), 0);
}

Example 5 with SupplierStreamlet

use of org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet in project heron by twitter.

the class StreamletImplTest method testSimpleBuild.

@Test
@SuppressWarnings("unchecked")
public void testSimpleBuild() throws Exception {
    Streamlet<String> baseStreamlet = builder.newSource(() -> "sa re ga ma");
    baseStreamlet.flatMap(x -> Arrays.asList(x.split(" "))).reduceByKeyAndWindow(x -> x, x -> 1, WindowConfig.TumblingCountWindow(10), (x, y) -> x + y);
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
    assertTrue(supplierStreamlet.isFullyBuilt());
    assertEquals(supplierStreamlet.getChildren().size(), 1);
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof FlatMapStreamlet);
    FlatMapStreamlet<String, String> fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof ReduceByKeyAndWindowStreamlet);
    ReduceByKeyAndWindowStreamlet<String, Integer, Integer> rStreamlet = (ReduceByKeyAndWindowStreamlet<String, Integer, Integer>) fStreamlet.getChildren().get(0);
    assertEquals(rStreamlet.getChildren().size(), 0);
}