Examples with TopologyBuilder org.apache.heron.api.topology.TopologyBuilder used on opensource projects

Example 31 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class LaunchRunnerTest method createTopology.

public static TopologyAPI.Topology createTopology(org.apache.heron.api.Config heronConfig) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout-1", new BaseRichSpout() {

        private static final long serialVersionUID = -762965195665496156L;

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }

        public void open(Map<String, Object> conf, TopologyContext context, SpoutOutputCollector collector) {
        }

        public void nextTuple() {
        }
    }, 2);
    builder.setBolt("bolt-1", new BaseBasicBolt() {

        private static final long serialVersionUID = -5738458486388778812L;

        public void execute(Tuple input, BasicOutputCollector collector) {
        }

        public void declareOutputFields(OutputFieldsDeclarer declarer) {
        }
    }, 1);
    HeronTopology heronTopology = builder.createTopology();
    return heronTopology.setName(TOPOLOGY_NAME).setConfig(heronConfig).setState(TopologyAPI.TopologyState.RUNNING).getTopology();
}

Example 32 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class StreamletImplTest method testCalculatedDefaultStageNames.

@Test
@SuppressWarnings("unchecked")
public void testCalculatedDefaultStageNames() {
    // create SupplierStreamlet
    Streamlet<String> baseStreamlet = builder.newSource(() -> "This is test content");
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertEquals(supplierStreamlet.getChildren().size(), 0);
    // apply the consumer function
    baseStreamlet.consume((SerializableConsumer<String>) s -> {
    });
    // build SupplierStreamlet
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
    // verify SupplierStreamlet
    assertTrue(supplierStreamlet.isFullyBuilt());
    assertEquals(1, supplierStreamlet.getChildren().size());
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof ConsumerStreamlet);
    assertEquals("consumer1", supplierStreamlet.getChildren().get(0).getName());
    // verify stageNames
    assertEquals(2, stageNames.size());
    List<String> expectedStageNames = Arrays.asList("consumer1", "supplier1");
    assertTrue(stageNames.containsAll(expectedStageNames));
    // verify ConsumerStreamlet
    ConsumerStreamlet<String> consumerStreamlet = (ConsumerStreamlet<String>) supplierStreamlet.getChildren().get(0);
    assertEquals(0, consumerStreamlet.getChildren().size());
}

Example 33 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class StreamletImplTest method testComplexBuild.

@Test
@SuppressWarnings("unchecked")
public void testComplexBuild() {
    // First source
    Streamlet<String> baseStreamlet1 = builder.newSource(() -> "sa re ga ma");
    Streamlet<String> leftStream = baseStreamlet1.flatMap(x -> Arrays.asList(x.split(" ")));
    // Second source
    Streamlet<String> baseStreamlet2 = builder.newSource(() -> "I Love You");
    Streamlet<String> rightStream = baseStreamlet2.flatMap(x -> Arrays.asList(x.split(" ")));
    // join
    leftStream.join(rightStream, x -> x, x -> x, WindowConfig.TumblingCountWindow(10), (x, y) -> x + y);
    SupplierStreamlet<String> supplierStreamlet1 = (SupplierStreamlet<String>) baseStreamlet1;
    SupplierStreamlet<String> supplierStreamlet2 = (SupplierStreamlet<String>) baseStreamlet2;
    assertFalse(supplierStreamlet1.isBuilt());
    assertFalse(supplierStreamlet2.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet1.build(topologyBuilder, stageNames);
    assertTrue(supplierStreamlet1.isBuilt());
    assertFalse(supplierStreamlet1.isFullyBuilt());
    supplierStreamlet2.build(topologyBuilder, stageNames);
    assertTrue(supplierStreamlet1.isFullyBuilt());
    assertTrue(supplierStreamlet2.isFullyBuilt());
    // go over all stuff
    assertEquals(supplierStreamlet1.getChildren().size(), 1);
    assertTrue(supplierStreamlet1.getChildren().get(0) instanceof FlatMapStreamlet);
    FlatMapStreamlet<String, String> fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet1.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof JoinStreamlet);
    JoinStreamlet<String, String, String, String> jStreamlet = (JoinStreamlet<String, String, String, String>) fStreamlet.getChildren().get(0);
    assertEquals(jStreamlet.getChildren().size(), 0);
    assertEquals(supplierStreamlet2.getChildren().size(), 1);
    assertTrue(supplierStreamlet2.getChildren().get(0) instanceof FlatMapStreamlet);
    fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet2.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof JoinStreamlet);
    jStreamlet = (JoinStreamlet<String, String, String, String>) fStreamlet.getChildren().get(0);
    assertEquals(jStreamlet.getChildren().size(), 0);
}

Example 34 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class StreamletImplTest method testSimpleBuild.

@Test
@SuppressWarnings("unchecked")
public void testSimpleBuild() throws Exception {
    Streamlet<String> baseStreamlet = builder.newSource(() -> "sa re ga ma");
    baseStreamlet.flatMap(x -> Arrays.asList(x.split(" "))).reduceByKeyAndWindow(x -> x, x -> 1, WindowConfig.TumblingCountWindow(10), (x, y) -> x + y);
    SupplierStreamlet<String> supplierStreamlet = (SupplierStreamlet<String>) baseStreamlet;
    assertFalse(supplierStreamlet.isBuilt());
    TopologyBuilder topologyBuilder = new TopologyBuilder();
    Set<String> stageNames = new HashSet<>();
    supplierStreamlet.build(topologyBuilder, stageNames);
    assertTrue(supplierStreamlet.isFullyBuilt());
    assertEquals(supplierStreamlet.getChildren().size(), 1);
    assertTrue(supplierStreamlet.getChildren().get(0) instanceof FlatMapStreamlet);
    FlatMapStreamlet<String, String> fStreamlet = (FlatMapStreamlet<String, String>) supplierStreamlet.getChildren().get(0);
    assertEquals(fStreamlet.getChildren().size(), 1);
    assertTrue(fStreamlet.getChildren().get(0) instanceof ReduceByKeyAndWindowStreamlet);
    ReduceByKeyAndWindowStreamlet<String, Integer, Integer> rStreamlet = (ReduceByKeyAndWindowStreamlet<String, Integer, Integer>) fStreamlet.getChildren().get(0);
    assertEquals(rStreamlet.getChildren().size(), 0);
}

Example 35 with TopologyBuilder

use of org.apache.heron.api.topology.TopologyBuilder in project heron by twitter.

the class BaseWindowedBoltTest method testSettingTumblingTimeWindow.

@Test
public void testSettingTumblingTimeWindow() {
    final Object[] args = new Object[] { -1L, 0L, 1L, 2L, 5L, 10L, null };
    for (Object arg : args) {
        TopologyBuilder builder = new TopologyBuilder();
        Object arg0 = arg;
        try {
            Duration windowLengthDuration = null;
            if (arg0 != null) {
                windowLengthDuration = Duration.ofMillis((Long) arg0);
            }
            builder.setBolt("testBolt", new TestBolt().withTumblingWindow(windowLengthDuration));
            if (arg0 == null) {
                fail(String.format("Window count duration cannot be null -- windowLengthDuration: %s", arg0));
            }
            if ((Long) arg0 <= 0) {
                fail(String.format("Window length cannot be zero or less -- windowLengthDuration: %s", arg0));
            }
        } catch (IllegalArgumentException e) {
            if (arg0 != null && (Long) arg0 > 0) {
                fail(String.format("Exception: %s thrown on valid input -- windowLengthDuration: %s", e.getMessage(), arg0));
            }
        }
    }
}