Examples with TopologyBuilder org.apache.storm.topology.TopologyBuilder used on opensource projects

Example 56 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class TopologyIntegrationTest method mkValidateTopology.

private StormTopology mkValidateTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("1", new TestWordSpout(true), 3);
    builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
    return builder.createTopology();
}

Example 57 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class TopologyIntegrationTest method testSubmitInactiveTopology.

@Test
public void testSubmitInactiveTopology() throws Exception {
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withDaemonConf(Collections.singletonMap(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true)).build()) {
        FeederSpout feeder = new FeederSpout(new Fields("field1"));
        AckFailMapTracker tracker = new AckFailMapTracker();
        feeder.setAckFailDelegate(tracker);
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", feeder);
        builder.setSpout("2", new OpenTrackedSpout());
        builder.setBolt("3", new PrepareTrackedBolt()).globalGrouping("1");
        boltPrepared = false;
        spoutOpened = false;
        StormTopology topology = builder.createTopology();
        cluster.submitTopologyWithOpts("test", Collections.singletonMap(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 10), topology, new SubmitOptions(TopologyInitialStatus.INACTIVE));
        cluster.advanceClusterTime(11);
        feeder.feed(new Values("a"), 1);
        cluster.advanceClusterTime(9);
        assertThat(boltPrepared, is(false));
        assertThat(spoutOpened, is(false));
        cluster.getNimbus().activate("test");
        cluster.advanceClusterTime(12);
        assertAcked(tracker, 1);
        assertThat(boltPrepared, is(true));
        assertThat(spoutOpened, is(true));
    }
}

Example 58 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class TopologyIntegrationTest method testComponentSpecificConfig.

@Test
public void testComponentSpecificConfig() throws Exception {
    Map<String, Object> daemonConf = new HashMap<>();
    daemonConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withDaemonConf(daemonConf).build()) {
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf")));
        Map<String, Object> componentConf = new HashMap<>();
        componentConf.put("fake.config", 123);
        componentConf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 20);
        componentConf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30);
        componentConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(Collections.singletonMap("fake.type", "bad.serializer"), Collections.singletonMap("fake.type2", "a.serializer")));
        topologyBuilder.setBolt("2", new TestConfBolt(componentConf)).shuffleGrouping("1").setMaxTaskParallelism(2).addConfiguration("fake.config2", 987);
        List<FixedTuple> testTuples = Arrays.asList("fake.config", Config.TOPOLOGY_MAX_TASK_PARALLELISM, Config.TOPOLOGY_MAX_SPOUT_PENDING, "fake.config2", Config.TOPOLOGY_KRYO_REGISTER).stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        Map<String, String> kryoRegister = new HashMap<>();
        kryoRegister.put("fake.type", "good.serializer");
        kryoRegister.put("fake.type3", "a.serializer3");
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(kryoRegister));
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams);
        Map<String, Object> expectedValues = new HashMap<>();
        expectedValues.put("fake.config", 123L);
        expectedValues.put("fake.config2", 987L);
        expectedValues.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 2L);
        expectedValues.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30L);
        Map<String, String> expectedKryoRegister = new HashMap<>();
        expectedKryoRegister.putAll(kryoRegister);
        expectedKryoRegister.put("fake.type2", "a.serializer");
        expectedValues.put(Config.TOPOLOGY_KRYO_REGISTER, expectedKryoRegister);
        List<Object> concatValues = Testing.readTuples(results, "2").stream().flatMap(values -> values.stream()).collect(Collectors.toList());
        assertThat(listToMap(concatValues), is(expectedValues));
    }
}

Example 59 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class TopologyIntegrationTest method testBasicTopology.

@ParameterizedTest
@ValueSource(strings = { "true", "false" })
public void testBasicTopology(boolean useLocalMessaging) throws Exception {
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(4).withDaemonConf(Collections.singletonMap(Config.STORM_LOCAL_MODE_ZMQ, !useLocalMessaging)).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 3);
        builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("word"));
        builder.setBolt("3", new TestGlobalCount()).globalGrouping("1");
        builder.setBolt("4", new TestAggregatesCounter()).globalGrouping("2");
        StormTopology topology = builder.createTopology();
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_WORKERS, 2);
        stormConf.put(Config.TOPOLOGY_TESTING_ALWAYS_TRY_SERIALIZE, true);
        List<FixedTuple> testTuples = Arrays.asList("nathan", "bob", "joey", "nathan").stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        assertThat(Testing.readTuples(results, "1"), containsInAnyOrder(new Values("nathan"), new Values("nathan"), new Values("bob"), new Values("joey")));
        assertThat(Testing.readTuples(results, "2"), containsInAnyOrder(new Values("nathan", 1), new Values("nathan", 2), new Values("bob", 1), new Values("joey", 1)));
        assertThat(Testing.readTuples(results, "3"), contains(new Values(1), new Values(2), new Values(3), new Values(4)));
        assertThat(Testing.readTuples(results, "4"), contains(new Values(1), new Values(2), new Values(3), new Values(4)));
    }
}

Example 60 with TopologyBuilder

use of org.apache.storm.topology.TopologyBuilder in project storm by apache.

the class TopologyIntegrationTest method mkInvalidateTopology1.

private StormTopology mkInvalidateTopology1() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("1", new TestWordSpout(true), 3);
    builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("3", new Fields("word"));
    return builder.createTopology();
}