Search in sources :

Example 1 with ConsumerStreamlet

use of org.apache.heron.streamlet.impl.streamlets.ConsumerStreamlet in project heron by twitter.

the class StreamletImplTest method testCalculatedDefaultStageNames.

@Test
@SuppressWarnings("unchecked")
public void testCalculatedDefaultStageNames() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertEquals(supplierStreamlet.getChildren().size(), 0);
    // apply the consumer function
    baseStreamlet.consume((SerializableConsumer<String>) s -> {
    });
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
    // verify SupplierStreamlet
    assertTrue(supplierStreamlet.isFullyBuilt());
    assertEquals(1, supplierStreamlet.getChildren().size());
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof ConsumerStreamlet);
    assertEquals("consumer1", supplierStreamlet.getChildren().get(0).getName());
    // verify stageNames
    assertEquals(2, stageNames.size());
    List<String> expectedStageNames = Arrays.asList("consumer1", "supplier1");
    assertTrue(stageNames.containsAll(expectedStageNames));
    // verify ConsumerStreamlet
    ConsumerStreamlet<String> consumerStreamlet = (ConsumerStreamlet<String>) supplierStreamlet.getChildren().get(0);
    assertEquals(0, consumerStreamlet.getChildren().size());
}
Also used : Arrays(java.util.Arrays) SpoutStreamlet(org.apache.heron.streamlet.impl.streamlets.SpoutStreamlet) WindowConfig(org.apache.heron.streamlet.WindowConfig) ShuffleStreamGrouping(org.apache.heron.api.grouping.ShuffleStreamGrouping) Builder(org.apache.heron.streamlet.Builder) CountByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.CountByKeyStreamlet) JoinStreamlet(org.apache.heron.streamlet.impl.streamlets.JoinStreamlet) Map(java.util.Map) Utils(org.apache.heron.api.utils.Utils) Assert.fail(org.junit.Assert.fail) StreamletReducers(org.apache.heron.streamlet.StreamletReducers) TestSpout(org.apache.heron.resource.TestSpout) SupplierStreamlet(org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet) TestBasicBolt(org.apache.heron.resource.TestBasicBolt) ReduceByKeyAndWindowStreamlet(org.apache.heron.streamlet.impl.streamlets.ReduceByKeyAndWindowStreamlet) IStreamletWindowOperator(org.apache.heron.streamlet.IStreamletWindowOperator) KeyByStreamlet(org.apache.heron.streamlet.impl.streamlets.KeyByStreamlet) SourceStreamlet(org.apache.heron.streamlet.impl.streamlets.SourceStreamlet) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) Collection(java.util.Collection) Set(java.util.Set) Context(org.apache.heron.streamlet.Context) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) SerializableTransformer(org.apache.heron.streamlet.SerializableTransformer) TopologyBuilder(org.apache.heron.api.topology.TopologyBuilder) KVStreamlet(org.apache.heron.streamlet.KVStreamlet) SerializablePredicate(org.apache.heron.streamlet.SerializablePredicate) CountByKeyAndWindowStreamlet(org.apache.heron.streamlet.impl.streamlets.CountByKeyAndWindowStreamlet) HashMap(java.util.HashMap) Function(java.util.function.Function) ConsumerStreamlet(org.apache.heron.streamlet.impl.streamlets.ConsumerStreamlet) HashSet(java.util.HashSet) UnionStreamlet(org.apache.heron.streamlet.impl.streamlets.UnionStreamlet) KVStreamletShadow(org.apache.heron.streamlet.impl.streamlets.KVStreamletShadow) FlatMapStreamlet(org.apache.heron.streamlet.impl.streamlets.FlatMapStreamlet) ReduceByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.ReduceByKeyStreamlet) MapStreamlet(org.apache.heron.streamlet.impl.streamlets.MapStreamlet) ByteAmount(org.apache.heron.common.basics.ByteAmount) Source(org.apache.heron.streamlet.Source) TransformStreamlet(org.apache.heron.streamlet.impl.streamlets.TransformStreamlet) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) TestBolt(org.apache.heron.resource.TestBolt) Config(org.apache.heron.streamlet.Config) Streamlet(org.apache.heron.streamlet.Streamlet) Consumer(java.util.function.Consumer) IStreamletRichOperator(org.apache.heron.streamlet.IStreamletRichOperator) CustomStreamlet(org.apache.heron.streamlet.impl.streamlets.CustomStreamlet) IStreamletBasicOperator(org.apache.heron.streamlet.IStreamletBasicOperator) TestWindowBolt(org.apache.heron.resource.TestWindowBolt) SerializableConsumer(org.apache.heron.streamlet.SerializableConsumer) GeneralReduceByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.GeneralReduceByKeyStreamlet) FilterStreamlet(org.apache.heron.streamlet.impl.streamlets.FilterStreamlet) Assert.assertEquals(org.junit.Assert.assertEquals) ConsumerStreamlet(org.apache.heron.streamlet.impl.streamlets.ConsumerStreamlet) TopologyBuilder(org.apache.heron.api.topology.TopologyBuilder) SupplierStreamlet(org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Set (java.util.Set)1 Consumer (java.util.function.Consumer)1 Function (java.util.function.Function)1 ShuffleStreamGrouping (org.apache.heron.api.grouping.ShuffleStreamGrouping)1 TopologyBuilder (org.apache.heron.api.topology.TopologyBuilder)1 Utils (org.apache.heron.api.utils.Utils)1 ByteAmount (org.apache.heron.common.basics.ByteAmount)1 TestBasicBolt (org.apache.heron.resource.TestBasicBolt)1 TestBolt (org.apache.heron.resource.TestBolt)1 TestSpout (org.apache.heron.resource.TestSpout)1 TestWindowBolt (org.apache.heron.resource.TestWindowBolt)1 Builder (org.apache.heron.streamlet.Builder)1 Config (org.apache.heron.streamlet.Config)1 Context (org.apache.heron.streamlet.Context)1