Examples with TestWordSpout org.apache.storm.testing.TestWordSpout used on opensource projects

Example 21 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class UtilsTest method testFindComponentCycles.

@Test
public void testFindComponentCycles() {
    class CycleDetectionScenario {

        final String testName;

        final String testDescription;

        final StormTopology topology;

        final int expectedCycles;

        CycleDetectionScenario() {
            testName = "dummy";
            testDescription = "dummy test";
            topology = null;
            expectedCycles = 0;
        }

        CycleDetectionScenario(String testName, String testDescription, StormTopology topology, int expectedCycles) {
            this.testName = testName.replace(' ', '-');
            this.testDescription = testDescription;
            this.topology = topology;
            this.expectedCycles = expectedCycles;
        }

        public List<CycleDetectionScenario> createTestScenarios() {
            List<CycleDetectionScenario> ret = new ArrayList<>();
            int testNo = 0;
            CycleDetectionScenario s;
            TopologyBuilder tb;
            // Base case
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                s = new CycleDetectionScenario(String.format("(%d) Base", testNo), "Three level component hierarchy with no loops", tb.createTopology(), 0);
                ret.add(s);
            }
            // single loop with one bolt
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                // loop bolt 3  (also connect bolt3 to spout 1)
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt3");
                ret.add(new CycleDetectionScenario(String.format("(%d) One Loop", testNo), "Three level component hierarchy with 1 cycle in bolt3", tb.createTopology(), 1));
            }
            // single loop with three bolts
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                // loop bolt 3 -> 4 -> 5 -> 3 (also connect bolt3 to spout1)
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt5");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3");
                tb.setBolt("bolt5", new TestWordCounter(), 10).shuffleGrouping("bolt4");
                ret.add(new CycleDetectionScenario(String.format("(%d) One Loop", testNo), "Four level component hierarchy with 1 cycle in bolt3,bolt4,bolt5", tb.createTopology(), 1));
            }
            // two loops with three bolts, and one bolt
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                // loop bolt 3 -> 4 -> 5 -> 3 (also connect bolt3 to spout1)
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt5");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3");
                tb.setBolt("bolt5", new TestWordCounter(), 10).shuffleGrouping("bolt4");
                // loop bolt 6  (also connect bolt6 to spout 1)
                tb.setBolt("bolt6", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt6");
                ret.add(new CycleDetectionScenario(String.format("(%d) Two Loops", testNo), "Four level component hierarchy with 2 cycles in bolt3,bolt4,bolt5 and bolt6", tb.createTopology(), 2));
            }
            // complex cycle
            {
                // (S1 -> B1 -> B2 -> B3 -> B4 <- S2), (B4 -> B3), (B4 -> B1)
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setSpout("spout2", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt4");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3").shuffleGrouping("spout2");
                ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#1", testNo), "Complex cycle (S1 -> B1 -> B2 -> B3 -> B4 <- S2), (B4 -> B3), (B4 -> B1)", tb.createTopology(), 1));
            }
            // another complex
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setSpout("spout2", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt4").shuffleGrouping("bolt2");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("spout2");
                ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#2", testNo), "Complex cycle 2 (S1 -> B1 <-> B2 -> B3 ), (S2 -> B4 -> B3), (B4 -> B1)", tb.createTopology(), 1));
            }
            // no spouts but with loops; the loops wont be detected
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("bolt4").shuffleGrouping("bolt2");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4");
                tb.setBolt("bolt4", new TestWordCounter(), 10);
                ret.add(new CycleDetectionScenario(String.format("(%d) No spout complex loops", testNo), "No Spouts, but with cycles (B1 <-> B2 -> B3 ), (B4 -> B3), (B4 -> B1)", tb.createTopology(), 0));
            }
            // now some randomly generated topologies
            int maxSpouts = 10;
            int maxBolts = 30;
            int randomTopoCnt = 100;
            for (int iRandTest = 0; iRandTest < randomTopoCnt; iRandTest++) {
                testNo++;
                tb = new TopologyBuilder();
                // topology component and connection counts
                int spoutCnt = ThreadLocalRandom.current().nextInt(0, maxSpouts) + 1;
                int boltCnt = ThreadLocalRandom.current().nextInt(0, maxBolts) + 1;
                int spoutToBoltConnectionCnt = ThreadLocalRandom.current().nextInt(spoutCnt * boltCnt) + 1;
                int boltToBoltConnectionCnt = ThreadLocalRandom.current().nextInt(boltCnt * boltCnt) + 1;
                Map<Integer, BoltDeclarer> boltDeclarers = new HashMap<>();
                for (int iSpout = 0; iSpout < spoutCnt; iSpout++) {
                    tb.setSpout("spout" + iSpout, new TestWordSpout(), 10);
                }
                for (int iBolt = 0; iBolt < boltCnt; iBolt++) {
                    boltDeclarers.put(iBolt, tb.setBolt("bolt" + iBolt, new TestWordCounter(), 10));
                }
                // spout to bolt connections
                for (int i = 0; i < spoutToBoltConnectionCnt; i++) {
                    int iSpout = ThreadLocalRandom.current().nextInt(0, spoutCnt);
                    int iBolt = ThreadLocalRandom.current().nextInt(0, boltCnt);
                    boltDeclarers.get(iBolt).shuffleGrouping("spout" + iSpout);
                }
                // bolt to bolt connections
                for (int i = 0; i < boltToBoltConnectionCnt; i++) {
                    int iBolt1 = ThreadLocalRandom.current().nextInt(0, boltCnt);
                    int iBolt2 = ThreadLocalRandom.current().nextInt(0, boltCnt);
                    boltDeclarers.get(iBolt2).shuffleGrouping("bolt" + iBolt1);
                }
                ret.add(new CycleDetectionScenario(String.format("(%d) Random Topo#%d", testNo, iRandTest), String.format("Random topology #%d, spouts=%d, bolts=%d, connections: fromSpouts=%d/fromBolts=%d", iRandTest, spoutCnt, boltCnt, spoutToBoltConnectionCnt, boltToBoltConnectionCnt), tb.createTopology(), -1));
            }
            return ret;
        }
    }
    List<String> testFailures = new ArrayList<>();
    new CycleDetectionScenario().createTestScenarios().forEach(x -> {
        LOG.info("==================== Running Test Scenario: {} =======================", x.testName);
        LOG.info("{}: {}", x.testName, x.testDescription);
        List<List<String>> loops = Utils.findComponentCycles(x.topology, x.testName);
        if (x.expectedCycles >= 0) {
            if (!loops.isEmpty()) {
                LOG.info("{} detected loops are \"{}\"", x.testName, loops.stream().map(y -> String.join(",", y)).collect(Collectors.joining(" ; ")));
            }
            if (loops.size() != x.expectedCycles) {
                testFailures.add(String.format("Test \"%s\" failed, detected cycles=%d does not match expected=%d for \"%s\"", x.testName, loops.size(), x.expectedCycles, x.testDescription));
                if (!loops.isEmpty()) {
                    testFailures.add(String.format("\t\tdetected loops are \"%s\"", loops.stream().map(y -> String.join(",", y)).collect(Collectors.joining(" ; "))));
                }
            }
        } else {
            // these are random topologies, with indeterminate number of loops
            LOG.info("{} detected loop count is \"{}\"", x.testName, loops.size());
        }
    });
    if (!testFailures.isEmpty()) {
        fail(String.join("\n", testFailures));
    }
}

Example 22 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class MessagingTest method testRemoteTransportWithManyTasksInReceivingExecutor.

@Test
public void testRemoteTransportWithManyTasksInReceivingExecutor() throws Exception {
    // STORM-3141 regression test
    // Verify that remote worker can handle many tasks in one executor
    Config topoConf = new Config();
    topoConf.put(Config.TOPOLOGY_WORKERS, 2);
    topoConf.put(Config.STORM_MESSAGING_TRANSPORT, "org.apache.storm.messaging.netty.Context");
    try (ILocalCluster cluster = new LocalCluster.Builder().withSimulatedTime().withSupervisors(1).withPortsPerSupervisor(2).withDaemonConf(topoConf).build()) {
        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("1", new TestWordSpout(true), 1);
        builder.setBolt("2", new TestGlobalCount(), 1).setNumTasks(10).shuffleGrouping("1");
        StormTopology stormTopology = builder.createTopology();
        List<FixedTuple> fixedTuples = new ArrayList<>();
        for (int i = 0; i < 12; i++) {
            fixedTuples.add(new FixedTuple(Collections.singletonList("a")));
            fixedTuples.add(new FixedTuple(Collections.singletonList("b")));
        }
        Map<String, List<FixedTuple>> data = new HashMap<>();
        data.put("1", fixedTuples);
        MockedSources mockedSources = new MockedSources(data);
        CompleteTopologyParam completeTopologyParam = new CompleteTopologyParam();
        completeTopologyParam.setMockedSources(mockedSources);
        Map<String, List<FixedTuple>> results = Testing.completeTopology(cluster, stormTopology, completeTopologyParam);
        Assert.assertEquals(6 * 4, Testing.readTuples(results, "2").size());
    }
}

Example 23 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class NimbusTest method testMemoryLoadLargerThanMaxHeapSize.

@Test
public void testMemoryLoadLargerThanMaxHeapSize() throws Exception {
    // Topology will not be able to be successfully scheduled: Config TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB=128.0 < 129.0,
    // Largest memory requirement of a component in the topology).
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 4);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.put(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN, "org.apache.storm.networktopography.DefaultRackDNSToSwitchMapping");
    config1.put(DaemonConfig.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, DefaultSchedulingPriorityStrategy.class.getName());
    config1.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, getDefaultResourceAwareStrategyClass().getName());
    config1.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 10.0);
    config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 0.0);
    config1.put(Config.TOPOLOGY_PRIORITY, 0);
    config1.put(Config.TOPOLOGY_SUBMITTER_USER, "zhuo");
    config1.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 128.0);
    config1.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 129.0);
    try {
        ServerUtils.validateTopologyWorkerMaxHeapSizeConfigs(config1, stormTopology1, 768.0);
        fail("Expected exception not thrown");
    } catch (InvalidTopologyException e) {
    // Expected...
    }
}

Example 24 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class ExecutorTransferMultiThreadingTest method createStormTopology.

private StormTopology createStormTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(sourceComp, new TestWordSpout(true), 1);
    builder.setBolt(destComp, new TestWordCounter(), 1).fieldsGrouping(sourceComp, new Fields("word"));
    return builder.createTopology();
}

Example 25 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class LocalNimbusTest method createTestTopology.

public static StormTopology createTestTopology() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("words", new TestWordSpout(), generateParallelismHint());
    builder.setBolt("count", new TestWordCounter(), generateParallelismHint()).shuffleGrouping("words");
    builder.setBolt("globalCount", new TestGlobalCount(), generateParallelismHint()).shuffleGrouping("count");
    return builder.createTopology();
}