use of org.apache.heron.streamlet.Streamlet in project heron by twitter.
the class StreamletImplTest method testStreamletNameIfDuplicateNameIsSet.
@Test(expected = RuntimeException.class)
public void testStreamletNameIfDuplicateNameIsSet() {
// create SupplierStreamlet
Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
// set duplicate streamlet name and expect thrown exception
supplierStreamlet.map((content) -> content.toUpperCase()).setName("MyMapStreamlet").map((content) -> content + "_test_suffix").setName("MyMapStreamlet");
// build SupplierStreamlet
assertFalse(supplierStreamlet.isBuilt());
TopologyBuilder topologyBuilder = new TopologyBuilder();
Set<String> stageNames = new HashSet<>();
supplierStreamlet.build(topologyBuilder, stageNames);
}
use of org.apache.heron.streamlet.Streamlet in project heron by twitter.
the class StreamletWithMapAndFlatMapAndFilterAndClone method buildTopology.
@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
Builder streamletBuilder = Builder.newBuilder();
Streamlet<String> streamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> SUMMER_MONTHS.contains(month.toLowerCase()) && incomingMonths.add(month.toLowerCase())).setName("summer-months").map((String word) -> word.substring(0, 3)).setName("summer-months-with-short-name");
List<Streamlet<String>> clonedStreamlet = streamlet.clone(2);
// Returns Summer Months with year
clonedStreamlet.get(0).map((String month) -> month + "_2018");
// Returns Summer Months with Uppercase
clonedStreamlet.get(1).map((String month) -> month.toUpperCase());
BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
return topology;
}
use of org.apache.heron.streamlet.Streamlet in project heron by twitter.
the class StreamletWithKeybyCountAndReduce method buildTopology.
@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
Builder streamletBuilder = Builder.newBuilder();
Streamlet<String> monthStreamlet = streamletBuilder.newSource(() -> MONTHS).setName("months-text").flatMap((String m) -> Arrays.asList(m.split(" - "))).setName("months").filter((month) -> incomingMonths.add(month.toLowerCase())).setName("unique-months");
SerializableFunction<String, String> getSeason = month -> {
if (SPRING_MONTHS.contains(month)) {
return "spring";
} else if (SUMMER_MONTHS.contains(month)) {
return "summer";
} else if (FALL_MONTHS.contains(month)) {
return "fall";
} else if (WINTER_MONTHS.contains(month)) {
return "winter";
} else {
return "really?";
}
};
SerializableFunction<String, Integer> getNumberOfDays = month -> {
switch(month) {
case "january":
return 31;
case "february":
// Dont use this code in real projects
return 28;
case "march":
return 31;
case "april":
return 30;
case "may":
return 31;
case "june":
return 30;
case "july":
return 31;
case "august":
return 31;
case "september":
return 30;
case "october":
return 31;
case "november":
return 30;
case "december":
return 31;
default:
// Shouldn't be here
return -1;
}
};
// Count months per season
monthStreamlet.keyBy(getSeason, getNumberOfDays).setName("key-by-season").countByKey(x -> x.getKey()).setName("key-by-and-count").map(x -> String.format("%s: %d months", x.getKey(), x.getValue())).setName("to-string");
// Sum days per season
monthStreamlet.<String, Integer>reduceByKey(getSeason, getNumberOfDays, StreamletReducers::sum).setName("sum-by-season").map(x -> String.format("%s: %d days", x.getKey(), x.getValue())).setName("to-string-2");
BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
return topology;
}
use of org.apache.heron.streamlet.Streamlet in project heron by twitter.
the class StreamletWithSplitAndWithStream method buildTopology.
@Override
protected TestTopologyBuilder buildTopology(TestTopologyBuilder testTopologyBuilder) {
Map<String, SerializablePredicate<Integer>> splitter = new HashMap();
splitter.put("all", i -> true);
splitter.put("positive", i -> i > 0);
splitter.put("negative", i -> i < 0);
Builder streamletBuilder = Builder.newBuilder();
Streamlet<Integer> multi = streamletBuilder.newSource(() -> atomicInteger.getAndIncrement()).setName("incremented-numbers-from--3").filter(i -> i <= 4).setName("numbers-lower-than-4").split(splitter).setName("split");
multi.withStream("all").map((Integer i) -> String.format("all_%d", i));
multi.withStream("positive").map((Integer i) -> String.format("pos_%d", i));
multi.withStream("negative").map((Integer i) -> String.format("neg_%d", i));
BuilderImpl streamletBuilderImpl = (BuilderImpl) streamletBuilder;
TestTopologyBuilder topology = (TestTopologyBuilder) streamletBuilderImpl.build(testTopologyBuilder);
return topology;
}
use of org.apache.heron.streamlet.Streamlet in project heron by twitter.
the class StreamletImplTest method testCalculatedDefaultStageNames.
@Test
@SuppressWarnings("unchecked")
public void testCalculatedDefaultStageNames() {
// create SupplierStreamlet
Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
assertEquals(supplierStreamlet.getChildren().size(), 0);
// apply the consumer function
baseStreamlet.consume((SerializableConsumer<String>) s -> {
});
// build SupplierStreamlet
assertFalse(supplierStreamlet.isBuilt());
TopologyBuilder topologyBuilder = new TopologyBuilder();
Set<String> stageNames = new HashSet<>();
supplierStreamlet.build(topologyBuilder, stageNames);
// verify SupplierStreamlet
assertTrue(supplierStreamlet.isFullyBuilt());
assertEquals(1, supplierStreamlet.getChildren().size());
assertTrue(supplierStreamlet.getChildren().get(0) instanceof ConsumerStreamlet);
assertEquals("consumer1", supplierStreamlet.getChildren().get(0).getName());
// verify stageNames
assertEquals(2, stageNames.size());
List<String> expectedStageNames = Arrays.asList("consumer1", "supplier1");
assertTrue(stageNames.containsAll(expectedStageNames));
// verify ConsumerStreamlet
ConsumerStreamlet<String> consumerStreamlet = (ConsumerStreamlet<String>) supplierStreamlet.getChildren().get(0);
assertEquals(0, consumerStreamlet.getChildren().size());
}
Aggregations