Search in sources :

Example 1 with TestPlannerSpout

use of org.apache.storm.testing.TestPlannerSpout in project storm by apache.

the class TopologyIntegrationTest method testComponentSpecificConfig.

@Test
public void testComponentSpecificConfig() throws Exception {
    Map<String, Object> daemonConf = new HashMap<>();
    daemonConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withDaemonConf(daemonConf).build()) {
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf")));
        Map<String, Object> componentConf = new HashMap<>();
        componentConf.put("fake.config", 123);
        componentConf.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 20);
        componentConf.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30);
        componentConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(Collections.singletonMap("fake.type", "bad.serializer"), Collections.singletonMap("fake.type2", "a.serializer")));
        topologyBuilder.setBolt("2", new TestConfBolt(componentConf)).shuffleGrouping("1").setMaxTaskParallelism(2).addConfiguration("fake.config2", 987);
        List<FixedTuple> testTuples = Arrays.asList("fake.config", Config.TOPOLOGY_MAX_TASK_PARALLELISM, Config.TOPOLOGY_MAX_SPOUT_PENDING, "fake.config2", Config.TOPOLOGY_KRYO_REGISTER).stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        Map<String, String> kryoRegister = new HashMap<>();
        kryoRegister.put("fake.type", "good.serializer");
        kryoRegister.put("fake.type3", "a.serializer3");
        Map<String, Object> stormConf = new HashMap<>();
        stormConf.put(Config.TOPOLOGY_KRYO_REGISTER, Arrays.asList(kryoRegister));
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(stormConf);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams);
        Map<String, Object> expectedValues = new HashMap<>();
        expectedValues.put("fake.config", 123L);
        expectedValues.put("fake.config2", 987L);
        expectedValues.put(Config.TOPOLOGY_MAX_TASK_PARALLELISM, 2L);
        expectedValues.put(Config.TOPOLOGY_MAX_SPOUT_PENDING, 30L);
        Map<String, String> expectedKryoRegister = new HashMap<>();
        expectedKryoRegister.putAll(kryoRegister);
        expectedKryoRegister.put("fake.type2", "a.serializer");
        expectedValues.put(Config.TOPOLOGY_KRYO_REGISTER, expectedKryoRegister);
        List<Object> concatValues = Testing.readTuples(results, "2").stream().flatMap(values -> values.stream()).collect(Collectors.toList());
        assertThat(listToMap(concatValues), is(expectedValues));
    }
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) OutputFieldsDeclarer(org.apache.storm.topology.OutputFieldsDeclarer) BaseRichSpout(org.apache.storm.topology.base.BaseRichSpout) Arrays(java.util.Arrays) TopologyContext(org.apache.storm.task.TopologyContext) SubmitOptions(org.apache.storm.generated.SubmitOptions) AssertLoop.assertFailed(org.apache.storm.AssertLoop.assertFailed) Assert.assertThat(org.junit.Assert.assertThat) Map(java.util.Map) TestAggregatesCounter(org.apache.storm.testing.TestAggregatesCounter) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) TestWordSpout(org.apache.storm.testing.TestWordSpout) MockedSources(org.apache.storm.testing.MockedSources) BaseRichBolt(org.apache.storm.topology.base.BaseRichBolt) EmitInfo(org.apache.storm.hooks.info.EmitInfo) FixedTuple(org.apache.storm.testing.FixedTuple) BoltAckInfo(org.apache.storm.hooks.info.BoltAckInfo) Fields(org.apache.storm.tuple.Fields) Collectors(java.util.stream.Collectors) TestWordCounter(org.apache.storm.testing.TestWordCounter) Test(org.junit.jupiter.api.Test) List(java.util.List) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) TrackedTopology(org.apache.storm.testing.TrackedTopology) Config(org.apache.storm.Config) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) TestGlobalCount(org.apache.storm.testing.TestGlobalCount) IntStream(java.util.stream.IntStream) TestConfBolt(org.apache.storm.testing.TestConfBolt) Testing(org.apache.storm.Testing) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) FeederSpout(org.apache.storm.testing.FeederSpout) Values(org.apache.storm.tuple.Values) Tuple(org.apache.storm.tuple.Tuple) OutputCollector(org.apache.storm.task.OutputCollector) StormTopology(org.apache.storm.generated.StormTopology) BoltFailInfo(org.apache.storm.hooks.info.BoltFailInfo) CompleteTopologyParam(org.apache.storm.testing.CompleteTopologyParam) BoltExecuteInfo(org.apache.storm.hooks.info.BoltExecuteInfo) TestPlannerSpout(org.apache.storm.testing.TestPlannerSpout) ValueSource(org.junit.jupiter.params.provider.ValueSource) AssertLoop.assertAcked(org.apache.storm.AssertLoop.assertAcked) AckFailMapTracker(org.apache.storm.testing.AckFailMapTracker) TopologyInitialStatus(org.apache.storm.generated.TopologyInitialStatus) BaseTaskHook(org.apache.storm.hooks.BaseTaskHook) Consumer(java.util.function.Consumer) LocalCluster(org.apache.storm.LocalCluster) IntegrationTest(org.apache.storm.testing.IntegrationTest) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) SpoutOutputCollector(org.apache.storm.spout.SpoutOutputCollector) Collections(java.util.Collections) LocalCluster(org.apache.storm.LocalCluster) HashMap(java.util.HashMap) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Values(org.apache.storm.tuple.Values) FixedTuple(org.apache.storm.testing.FixedTuple) TestConfBolt(org.apache.storm.testing.TestConfBolt) MockedSources(org.apache.storm.testing.MockedSources) Fields(org.apache.storm.tuple.Fields) CompleteTopologyParam(org.apache.storm.testing.CompleteTopologyParam) List(java.util.List) ArrayList(java.util.ArrayList) TestPlannerSpout(org.apache.storm.testing.TestPlannerSpout) Test(org.junit.jupiter.api.Test) IntegrationTest(org.apache.storm.testing.IntegrationTest) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 2 with TestPlannerSpout

use of org.apache.storm.testing.TestPlannerSpout in project storm by apache.

the class TopologyIntegrationTest method testKryoDecoratorsConfig.

@Test
public void testKryoDecoratorsConfig() throws Exception {
    Map<String, Object> daemonConf = new HashMap<>();
    daemonConf.put(Config.TOPOLOGY_SKIP_MISSING_KRYO_REGISTRATIONS, true);
    daemonConf.put(Config.TOPOLOGY_KRYO_DECORATORS, "this-is-overridden");
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withDaemonConf(daemonConf).build()) {
        TopologyBuilder topologyBuilder = new TopologyBuilder();
        topologyBuilder.setSpout("1", new TestPlannerSpout(new Fields("conf")));
        topologyBuilder.setBolt("2", new TestConfBolt(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "two")))).shuffleGrouping("1");
        List<FixedTuple> testTuples = Arrays.asList(new Values(Config.TOPOLOGY_KRYO_DECORATORS)).stream().map(value -> new FixedTuple(value)).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        completeTopologyParams.setStormConf(Collections.singletonMap(Config.TOPOLOGY_KRYO_DECORATORS, Arrays.asList("one", "three")));
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topologyBuilder.createTopology(), completeTopologyParams);
        List<Object> concatValues = Testing.readTuples(results, "2").stream().flatMap(values -> values.stream()).collect(Collectors.toList());
        assertThat(concatValues.get(0), is(Config.TOPOLOGY_KRYO_DECORATORS));
        assertThat(concatValues.get(1), is(Arrays.asList("one", "two", "three")));
    }
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) OutputFieldsDeclarer(org.apache.storm.topology.OutputFieldsDeclarer) BaseRichSpout(org.apache.storm.topology.base.BaseRichSpout) Arrays(java.util.Arrays) TopologyContext(org.apache.storm.task.TopologyContext) SubmitOptions(org.apache.storm.generated.SubmitOptions) AssertLoop.assertFailed(org.apache.storm.AssertLoop.assertFailed) Assert.assertThat(org.junit.Assert.assertThat) Map(java.util.Map) TestAggregatesCounter(org.apache.storm.testing.TestAggregatesCounter) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) TestWordSpout(org.apache.storm.testing.TestWordSpout) MockedSources(org.apache.storm.testing.MockedSources) BaseRichBolt(org.apache.storm.topology.base.BaseRichBolt) EmitInfo(org.apache.storm.hooks.info.EmitInfo) FixedTuple(org.apache.storm.testing.FixedTuple) BoltAckInfo(org.apache.storm.hooks.info.BoltAckInfo) Fields(org.apache.storm.tuple.Fields) Collectors(java.util.stream.Collectors) TestWordCounter(org.apache.storm.testing.TestWordCounter) Test(org.junit.jupiter.api.Test) List(java.util.List) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) TrackedTopology(org.apache.storm.testing.TrackedTopology) Config(org.apache.storm.Config) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) TestGlobalCount(org.apache.storm.testing.TestGlobalCount) IntStream(java.util.stream.IntStream) TestConfBolt(org.apache.storm.testing.TestConfBolt) Testing(org.apache.storm.Testing) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) FeederSpout(org.apache.storm.testing.FeederSpout) Values(org.apache.storm.tuple.Values) Tuple(org.apache.storm.tuple.Tuple) OutputCollector(org.apache.storm.task.OutputCollector) StormTopology(org.apache.storm.generated.StormTopology) BoltFailInfo(org.apache.storm.hooks.info.BoltFailInfo) CompleteTopologyParam(org.apache.storm.testing.CompleteTopologyParam) BoltExecuteInfo(org.apache.storm.hooks.info.BoltExecuteInfo) TestPlannerSpout(org.apache.storm.testing.TestPlannerSpout) ValueSource(org.junit.jupiter.params.provider.ValueSource) AssertLoop.assertAcked(org.apache.storm.AssertLoop.assertAcked) AckFailMapTracker(org.apache.storm.testing.AckFailMapTracker) TopologyInitialStatus(org.apache.storm.generated.TopologyInitialStatus) BaseTaskHook(org.apache.storm.hooks.BaseTaskHook) Consumer(java.util.function.Consumer) LocalCluster(org.apache.storm.LocalCluster) IntegrationTest(org.apache.storm.testing.IntegrationTest) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) SpoutOutputCollector(org.apache.storm.spout.SpoutOutputCollector) Collections(java.util.Collections) LocalCluster(org.apache.storm.LocalCluster) HashMap(java.util.HashMap) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) Values(org.apache.storm.tuple.Values) FixedTuple(org.apache.storm.testing.FixedTuple) TestConfBolt(org.apache.storm.testing.TestConfBolt) MockedSources(org.apache.storm.testing.MockedSources) Fields(org.apache.storm.tuple.Fields) CompleteTopologyParam(org.apache.storm.testing.CompleteTopologyParam) List(java.util.List) ArrayList(java.util.ArrayList) TestPlannerSpout(org.apache.storm.testing.TestPlannerSpout) Test(org.junit.jupiter.api.Test) IntegrationTest(org.apache.storm.testing.IntegrationTest) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 3 with TestPlannerSpout

use of org.apache.storm.testing.TestPlannerSpout in project storm by apache.

the class TopologyIntegrationTest method testHooks.

@Test
public void testHooks() throws Exception {
    try (LocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestPlannerSpout(new Fields("conf")));
        builder.setBolt("2", new HooksBolt()).shuffleGrouping("1");
        StormTopology topology = builder.createTopology();
        List<FixedTuple> testTuples = Arrays.asList(1, 1, 1, 1).stream().map(value -> new FixedTuple(new Values(value))).collect(Collectors.toList());
        MockedSources mockedSources = new MockedSources(Collections.singletonMap("1", testTuples));
        CompleteTopologyParam completeTopologyParams = new CompleteTopologyParam();
        completeTopologyParams.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, topology, completeTopologyParams);
        List<List<Object>> expectedTuples = Arrays.asList(Arrays.asList(0, 0, 0, 0), Arrays.asList(2, 1, 0, 1), Arrays.asList(4, 1, 1, 2), Arrays.asList(6, 2, 1, 3));
        assertThat(Testing.readTuples(results, "2"), is(expectedTuples));
    }
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) OutputFieldsDeclarer(org.apache.storm.topology.OutputFieldsDeclarer) BaseRichSpout(org.apache.storm.topology.base.BaseRichSpout) Arrays(java.util.Arrays) TopologyContext(org.apache.storm.task.TopologyContext) SubmitOptions(org.apache.storm.generated.SubmitOptions) AssertLoop.assertFailed(org.apache.storm.AssertLoop.assertFailed) Assert.assertThat(org.junit.Assert.assertThat) Map(java.util.Map) TestAggregatesCounter(org.apache.storm.testing.TestAggregatesCounter) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) TestWordSpout(org.apache.storm.testing.TestWordSpout) MockedSources(org.apache.storm.testing.MockedSources) BaseRichBolt(org.apache.storm.topology.base.BaseRichBolt) EmitInfo(org.apache.storm.hooks.info.EmitInfo) FixedTuple(org.apache.storm.testing.FixedTuple) BoltAckInfo(org.apache.storm.hooks.info.BoltAckInfo) Fields(org.apache.storm.tuple.Fields) Collectors(java.util.stream.Collectors) TestWordCounter(org.apache.storm.testing.TestWordCounter) Test(org.junit.jupiter.api.Test) List(java.util.List) Matchers.contains(org.hamcrest.Matchers.contains) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) TrackedTopology(org.apache.storm.testing.TrackedTopology) Config(org.apache.storm.Config) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) TestGlobalCount(org.apache.storm.testing.TestGlobalCount) IntStream(java.util.stream.IntStream) TestConfBolt(org.apache.storm.testing.TestConfBolt) Testing(org.apache.storm.Testing) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) FeederSpout(org.apache.storm.testing.FeederSpout) Values(org.apache.storm.tuple.Values) Tuple(org.apache.storm.tuple.Tuple) OutputCollector(org.apache.storm.task.OutputCollector) StormTopology(org.apache.storm.generated.StormTopology) BoltFailInfo(org.apache.storm.hooks.info.BoltFailInfo) CompleteTopologyParam(org.apache.storm.testing.CompleteTopologyParam) BoltExecuteInfo(org.apache.storm.hooks.info.BoltExecuteInfo) TestPlannerSpout(org.apache.storm.testing.TestPlannerSpout) ValueSource(org.junit.jupiter.params.provider.ValueSource) AssertLoop.assertAcked(org.apache.storm.AssertLoop.assertAcked) AckFailMapTracker(org.apache.storm.testing.AckFailMapTracker) TopologyInitialStatus(org.apache.storm.generated.TopologyInitialStatus) BaseTaskHook(org.apache.storm.hooks.BaseTaskHook) Consumer(java.util.function.Consumer) LocalCluster(org.apache.storm.LocalCluster) IntegrationTest(org.apache.storm.testing.IntegrationTest) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) SpoutOutputCollector(org.apache.storm.spout.SpoutOutputCollector) Collections(java.util.Collections) LocalCluster(org.apache.storm.LocalCluster) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) StormTopology(org.apache.storm.generated.StormTopology) Values(org.apache.storm.tuple.Values) FixedTuple(org.apache.storm.testing.FixedTuple) MockedSources(org.apache.storm.testing.MockedSources) Fields(org.apache.storm.tuple.Fields) CompleteTopologyParam(org.apache.storm.testing.CompleteTopologyParam) List(java.util.List) ArrayList(java.util.ArrayList) TestPlannerSpout(org.apache.storm.testing.TestPlannerSpout) Test(org.junit.jupiter.api.Test) IntegrationTest(org.apache.storm.testing.IntegrationTest) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

ArrayList (java.util.ArrayList)3 Arrays (java.util.Arrays)3 Collections (java.util.Collections)3 HashMap (java.util.HashMap)3 List (java.util.List)3 Map (java.util.Map)3 Consumer (java.util.function.Consumer)3 Collectors (java.util.stream.Collectors)3 IntStream (java.util.stream.IntStream)3 AssertLoop.assertAcked (org.apache.storm.AssertLoop.assertAcked)3 AssertLoop.assertFailed (org.apache.storm.AssertLoop.assertFailed)3 Config (org.apache.storm.Config)3 LocalCluster (org.apache.storm.LocalCluster)3 Testing (org.apache.storm.Testing)3 InvalidTopologyException (org.apache.storm.generated.InvalidTopologyException)3 StormTopology (org.apache.storm.generated.StormTopology)3 SubmitOptions (org.apache.storm.generated.SubmitOptions)3 TopologyInitialStatus (org.apache.storm.generated.TopologyInitialStatus)3 BaseTaskHook (org.apache.storm.hooks.BaseTaskHook)3 BoltAckInfo (org.apache.storm.hooks.info.BoltAckInfo)3