Search in sources :

Example 1 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class StreamletImplTest method testStreamletNameIfDuplicateNameIsSet.

@Test(expected = RuntimeException.class)
public void testStreamletNameIfDuplicateNameIsSet() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    // set duplicate streamlet name and expect thrown exception
    supplierStreamlet.map((content) -> content.toUpperCase()).setName("MyMapStreamlet").map((content) -> content + "_test_suffix").setName("MyMapStreamlet");
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
}
Also used : Arrays(java.util.Arrays) SpoutStreamlet(org.apache.heron.streamlet.impl.streamlets.SpoutStreamlet) WindowConfig(org.apache.heron.streamlet.WindowConfig) ShuffleStreamGrouping(org.apache.heron.api.grouping.ShuffleStreamGrouping) Builder(org.apache.heron.streamlet.Builder) CountByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.CountByKeyStreamlet) JoinStreamlet(org.apache.heron.streamlet.impl.streamlets.JoinStreamlet) Map(java.util.Map) Utils(org.apache.heron.api.utils.Utils) Assert.fail(org.junit.Assert.fail) StreamletReducers(org.apache.heron.streamlet.StreamletReducers) TestSpout(org.apache.heron.resource.TestSpout) SupplierStreamlet(org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet) TestBasicBolt(org.apache.heron.resource.TestBasicBolt) ReduceByKeyAndWindowStreamlet(org.apache.heron.streamlet.impl.streamlets.ReduceByKeyAndWindowStreamlet) IStreamletWindowOperator(org.apache.heron.streamlet.IStreamletWindowOperator) KeyByStreamlet(org.apache.heron.streamlet.impl.streamlets.KeyByStreamlet) SourceStreamlet(org.apache.heron.streamlet.impl.streamlets.SourceStreamlet) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) Collection(java.util.Collection) Set(java.util.Set) Context(org.apache.heron.streamlet.Context) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) SerializableTransformer(org.apache.heron.streamlet.SerializableTransformer) TopologyBuilder(org.apache.heron.api.topology.TopologyBuilder) KVStreamlet(org.apache.heron.streamlet.KVStreamlet) SerializablePredicate(org.apache.heron.streamlet.SerializablePredicate) CountByKeyAndWindowStreamlet(org.apache.heron.streamlet.impl.streamlets.CountByKeyAndWindowStreamlet) HashMap(java.util.HashMap) Function(java.util.function.Function) ConsumerStreamlet(org.apache.heron.streamlet.impl.streamlets.ConsumerStreamlet) HashSet(java.util.HashSet) UnionStreamlet(org.apache.heron.streamlet.impl.streamlets.UnionStreamlet) KVStreamletShadow(org.apache.heron.streamlet.impl.streamlets.KVStreamletShadow) FlatMapStreamlet(org.apache.heron.streamlet.impl.streamlets.FlatMapStreamlet) ReduceByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.ReduceByKeyStreamlet) MapStreamlet(org.apache.heron.streamlet.impl.streamlets.MapStreamlet) ByteAmount(org.apache.heron.common.basics.ByteAmount) Source(org.apache.heron.streamlet.Source) TransformStreamlet(org.apache.heron.streamlet.impl.streamlets.TransformStreamlet) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) TestBolt(org.apache.heron.resource.TestBolt) Config(org.apache.heron.streamlet.Config) Streamlet(org.apache.heron.streamlet.Streamlet) Consumer(java.util.function.Consumer) IStreamletRichOperator(org.apache.heron.streamlet.IStreamletRichOperator) CustomStreamlet(org.apache.heron.streamlet.impl.streamlets.CustomStreamlet) IStreamletBasicOperator(org.apache.heron.streamlet.IStreamletBasicOperator) TestWindowBolt(org.apache.heron.resource.TestWindowBolt) SerializableConsumer(org.apache.heron.streamlet.SerializableConsumer) GeneralReduceByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.GeneralReduceByKeyStreamlet) FilterStreamlet(org.apache.heron.streamlet.impl.streamlets.FilterStreamlet) Assert.assertEquals(org.junit.Assert.assertEquals) TopologyBuilder(org.apache.heron.api.topology.TopologyBuilder) SupplierStreamlet(org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 2 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class StreamletWithMapAndFlatMapAndFilterAndClone method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<String> streamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> SUMMER_MONTHS.contains(month.toLowerCase()) && incomingMonths.add(month.toLowerCase())).setName("summer-months").map((String word) -> word.substring(0, 3)).setName("summer-months-with-short-name");
    List<Streamlet<String>> clonedStreamlet = streamlet.clone(2);
    // Returns Summer Months with year
    clonedStreamlet.get(0).map((String month) -> month + "_2018");
    // Returns Summer Months with Uppercase
    clonedStreamlet.get(1).map((String month) -> month.toUpperCase());
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}
Also used : HashSet(java.util.HashSet) Arrays(java.util.Arrays) List(java.util.List) MalformedURLException(java.net.MalformedURLException) Config(org.apache.heron.api.Config) AbstractTestTopology(org.apache.heron.integration_test.common.AbstractTestTopology) Set(java.util.Set) Builder(org.apache.heron.streamlet.Builder) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder) BuilderImpl(org.apache.heron.streamlet.impl.BuilderImpl) Streamlet(org.apache.heron.streamlet.Streamlet) BuilderImpl(org.apache.heron.streamlet.impl.BuilderImpl) Builder(org.apache.heron.streamlet.Builder) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder) Streamlet(org.apache.heron.streamlet.Streamlet) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder)

Example 3 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class StreamletWithKeybyCountAndReduce method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<String> monthStreamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> incomingMonths.add(month.toLowerCase())).setName("unique-months");
    SerializableFunction<String, String> getSeason = month -> {
        if (SPRING_MONTHS.contains(month)) {
            return "spring";
        } else if (SUMMER_MONTHS.contains(month)) {
            return "summer";
        } else if (FALL_MONTHS.contains(month)) {
            return "fall";
        } else if (WINTER_MONTHS.contains(month)) {
            return "winter";
        } else {
            return "really?";
        }
    };
    SerializableFunction<String, Integer> getNumberOfDays = month -> {
        switch(month) {
            case "january":
                return 31;
            case "february":
                // Dont use this code in real projects
                return 28;
            case "march":
                return 31;
            case "april":
                return 30;
            case "may":
                return 31;
            case "june":
                return 30;
            case "july":
                return 31;
            case "august":
                return 31;
            case "september":
                return 30;
            case "october":
                return 31;
            case "november":
                return 30;
            case "december":
                return 31;
            default:
                // Shouldn't be here
                return -1;
        }
    };
    // Count months per season
    monthStreamlet.keyBy(getSeason, getNumberOfDays).setName("key-by-season").countByKey(x -> x.getKey()).setName("key-by-and-count").map(x -> String.format("%s: %d months", x.getKey(), x.getValue())).setName("to-string");
    // Sum days per season
    monthStreamlet.<String, Integer>reduceByKey(getSeason, getNumberOfDays, StreamletReducers::sum).setName("sum-by-season").map(x -> String.format("%s: %d days", x.getKey(), x.getValue())).setName("to-string-2");
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}
Also used : HashSet(java.util.HashSet) SerializableFunction(org.apache.heron.streamlet.SerializableFunction) Arrays(java.util.Arrays) MalformedURLException(java.net.MalformedURLException) Config(org.apache.heron.api.Config) AbstractTestTopology(org.apache.heron.integration_test.common.AbstractTestTopology) Set(java.util.Set) Builder(org.apache.heron.streamlet.Builder) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder) StreamletReducers(org.apache.heron.streamlet.StreamletReducers) BuilderImpl(org.apache.heron.streamlet.impl.BuilderImpl) Streamlet(org.apache.heron.streamlet.Streamlet) BuilderImpl(org.apache.heron.streamlet.impl.BuilderImpl) Builder(org.apache.heron.streamlet.Builder) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder)

Example 4 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class StreamletWithSplitAndWithStream method buildTopology.

@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
    Map<String, SerializablePredicate<Integer>> splitter = new HashMap();
    splitter.put("all", i -> true);
    splitter.put("positive", i -> i > 0);
    splitter.put("negative", i -> i < 0);
    Builder streamletBuilder = Builder.newBuilder();
    Streamlet<Integer> multi = streamletBuilder.newSource(() -> atomicInteger.getAndIncrement()).setName("incremented-numbers-from--3").filter(i -> i <= 4).setName("numbers-lower-than-4").split(splitter).setName("split");
    multi.withStream("all").map((Integer i) -> String.format("all_%d", i));
    multi.withStream("positive").map((Integer i) -> String.format("pos_%d", i));
    multi.withStream("negative").map((Integer i) -> String.format("neg_%d", i));
    BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
    TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
    return topology;
}
Also used : AtomicInteger(java.util.concurrent.atomic.AtomicInteger) SerializablePredicate(org.apache.heron.streamlet.SerializablePredicate) MalformedURLException(java.net.MalformedURLException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) Config(org.apache.heron.api.Config) AbstractTestTopology(org.apache.heron.integration_test.common.AbstractTestTopology) HashMap(java.util.HashMap) Builder(org.apache.heron.streamlet.Builder) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder) BuilderImpl(org.apache.heron.streamlet.impl.BuilderImpl) Streamlet(org.apache.heron.streamlet.Streamlet) BuilderImpl(org.apache.heron.streamlet.impl.BuilderImpl) HashMap(java.util.HashMap) Builder(org.apache.heron.streamlet.Builder) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder) SerializablePredicate(org.apache.heron.streamlet.SerializablePredicate) TestTopologyBuilder(org.apache.heron.integration_test.core.TestTopologyBuilder)

Example 5 with Streamlet

use of org.apache.heron.streamlet.Streamlet in project heron by twitter.

the class StreamletImplTest method testCalculatedDefaultStageNames.

@Test
@SuppressWarnings("unchecked")
public void testCalculatedDefaultStageNames() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertEquals(supplierStreamlet.getChildren().size(), 0);
    // apply the consumer function
    baseStreamlet.consume((SerializableConsumer<String>) s -> {
    });
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
    // verify SupplierStreamlet
    assertTrue(supplierStreamlet.isFullyBuilt());
    assertEquals(1, supplierStreamlet.getChildren().size());
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof ConsumerStreamlet);
    assertEquals("consumer1", supplierStreamlet.getChildren().get(0).getName());
    // verify stageNames
    assertEquals(2, stageNames.size());
    List<String> expectedStageNames = Arrays.asList("consumer1", "supplier1");
    assertTrue(stageNames.containsAll(expectedStageNames));
    // verify ConsumerStreamlet
    ConsumerStreamlet<String> consumerStreamlet = (ConsumerStreamlet<String>) supplierStreamlet.getChildren().get(0);
    assertEquals(0, consumerStreamlet.getChildren().size());
}
Also used : Arrays(java.util.Arrays) SpoutStreamlet(org.apache.heron.streamlet.impl.streamlets.SpoutStreamlet) WindowConfig(org.apache.heron.streamlet.WindowConfig) ShuffleStreamGrouping(org.apache.heron.api.grouping.ShuffleStreamGrouping) Builder(org.apache.heron.streamlet.Builder) CountByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.CountByKeyStreamlet) JoinStreamlet(org.apache.heron.streamlet.impl.streamlets.JoinStreamlet) Map(java.util.Map) Utils(org.apache.heron.api.utils.Utils) Assert.fail(org.junit.Assert.fail) StreamletReducers(org.apache.heron.streamlet.StreamletReducers) TestSpout(org.apache.heron.resource.TestSpout) SupplierStreamlet(org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet) TestBasicBolt(org.apache.heron.resource.TestBasicBolt) ReduceByKeyAndWindowStreamlet(org.apache.heron.streamlet.impl.streamlets.ReduceByKeyAndWindowStreamlet) IStreamletWindowOperator(org.apache.heron.streamlet.IStreamletWindowOperator) KeyByStreamlet(org.apache.heron.streamlet.impl.streamlets.KeyByStreamlet) SourceStreamlet(org.apache.heron.streamlet.impl.streamlets.SourceStreamlet) KeyedWindow(org.apache.heron.streamlet.KeyedWindow) Collection(java.util.Collection) Set(java.util.Set) Context(org.apache.heron.streamlet.Context) List(java.util.List) Assert.assertFalse(org.junit.Assert.assertFalse) SerializableTransformer(org.apache.heron.streamlet.SerializableTransformer) TopologyBuilder(org.apache.heron.api.topology.TopologyBuilder) KVStreamlet(org.apache.heron.streamlet.KVStreamlet) SerializablePredicate(org.apache.heron.streamlet.SerializablePredicate) CountByKeyAndWindowStreamlet(org.apache.heron.streamlet.impl.streamlets.CountByKeyAndWindowStreamlet) HashMap(java.util.HashMap) Function(java.util.function.Function) ConsumerStreamlet(org.apache.heron.streamlet.impl.streamlets.ConsumerStreamlet) HashSet(java.util.HashSet) UnionStreamlet(org.apache.heron.streamlet.impl.streamlets.UnionStreamlet) KVStreamletShadow(org.apache.heron.streamlet.impl.streamlets.KVStreamletShadow) FlatMapStreamlet(org.apache.heron.streamlet.impl.streamlets.FlatMapStreamlet) ReduceByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.ReduceByKeyStreamlet) MapStreamlet(org.apache.heron.streamlet.impl.streamlets.MapStreamlet) ByteAmount(org.apache.heron.common.basics.ByteAmount) Source(org.apache.heron.streamlet.Source) TransformStreamlet(org.apache.heron.streamlet.impl.streamlets.TransformStreamlet) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) TestBolt(org.apache.heron.resource.TestBolt) Config(org.apache.heron.streamlet.Config) Streamlet(org.apache.heron.streamlet.Streamlet) Consumer(java.util.function.Consumer) IStreamletRichOperator(org.apache.heron.streamlet.IStreamletRichOperator) CustomStreamlet(org.apache.heron.streamlet.impl.streamlets.CustomStreamlet) IStreamletBasicOperator(org.apache.heron.streamlet.IStreamletBasicOperator) TestWindowBolt(org.apache.heron.resource.TestWindowBolt) SerializableConsumer(org.apache.heron.streamlet.SerializableConsumer) GeneralReduceByKeyStreamlet(org.apache.heron.streamlet.impl.streamlets.GeneralReduceByKeyStreamlet) FilterStreamlet(org.apache.heron.streamlet.impl.streamlets.FilterStreamlet) Assert.assertEquals(org.junit.Assert.assertEquals) ConsumerStreamlet(org.apache.heron.streamlet.impl.streamlets.ConsumerStreamlet) TopologyBuilder(org.apache.heron.api.topology.TopologyBuilder) SupplierStreamlet(org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

Builder (org.apache.heron.streamlet.Builder)8 Streamlet (org.apache.heron.streamlet.Streamlet)8 Arrays (java.util.Arrays)6 HashSet (java.util.HashSet)5 List (java.util.List)5 Set (java.util.Set)5 Config (org.apache.heron.streamlet.Config)5 HashMap (java.util.HashMap)4 Map (java.util.Map)4 SerializablePredicate (org.apache.heron.streamlet.SerializablePredicate)4 StreamletReducers (org.apache.heron.streamlet.StreamletReducers)4 WindowConfig (org.apache.heron.streamlet.WindowConfig)4 Collection (java.util.Collection)3 Consumer (java.util.function.Consumer)3 Function (java.util.function.Function)3 ShuffleStreamGrouping (org.apache.heron.api.grouping.ShuffleStreamGrouping)3 TopologyBuilder (org.apache.heron.api.topology.TopologyBuilder)3 Utils (org.apache.heron.api.utils.Utils)3 ByteAmount (org.apache.heron.common.basics.ByteAmount)3 TestBasicBolt (org.apache.heron.resource.TestBasicBolt)3