Examples with BoltDeclarer org.apache.storm.topology.BoltDeclarer used on opensource projects

Example 26 with BoltDeclarer

use of org.apache.storm.topology.BoltDeclarer in project storm by apache.

the class StreamBuilder method addStatefulBolt.

private StreamBolt addStatefulBolt(TopologyBuilder topologyBuilder, String boltId, Set<ProcessorNode> initialProcessors, List<ProcessorNode> group) {
    StateQueryProcessor<?, ?> stateQueryProcessor = getStateQueryProcessor(group);
    StatefulProcessorBolt<?, ?> bolt;
    if (stateQueryProcessor == null) {
        bolt = new StatefulProcessorBolt<>(boltId, graph, group);
        BoltDeclarer boltDeclarer = topologyBuilder.setBolt(boltId, bolt, getParallelism(group));
        bolt.setStreamToInitialProcessors(wireBolt(group, boltDeclarer, initialProcessors));
        streamBolts.put(bolt, boltDeclarer);
    } else {
        // state query is added to the existing stateful bolt
        ProcessorNode updateStateNode = stateQueryProcessor.getStreamState().getUpdateStateNode();
        bolt = findStatefulProcessorBolt(updateStateNode);
        for (ProcessorNode node : group) {
            node.setComponentId(bolt.getId());
        }
        bolt.addNodes(group);
        bolt.addStreamToInitialProcessors(wireBolt(bolt.getNodes(), streamBolts.get(bolt), initialProcessors));
    }
    return bolt;
}

Example 27 with BoltDeclarer

use of org.apache.storm.topology.BoltDeclarer in project storm by apache.

the class UtilsTest method testFindComponentCycles.

@Test
public void testFindComponentCycles() {
    class CycleDetectionScenario {

        final String testName;

        final String testDescription;

        final StormTopology topology;

        final int expectedCycles;

        CycleDetectionScenario() {
            testName = "dummy";
            testDescription = "dummy test";
            topology = null;
            expectedCycles = 0;
        }

        CycleDetectionScenario(String testName, String testDescription, StormTopology topology, int expectedCycles) {
            this.testName = testName.replace(' ', '-');
            this.testDescription = testDescription;
            this.topology = topology;
            this.expectedCycles = expectedCycles;
        }

        public List<CycleDetectionScenario> createTestScenarios() {
            List<CycleDetectionScenario> ret = new ArrayList<>();
            int testNo = 0;
            CycleDetectionScenario s;
            TopologyBuilder tb;
            // Base case
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                s = new CycleDetectionScenario(String.format("(%d) Base", testNo), "Three level component hierarchy with no loops", tb.createTopology(), 0);
                ret.add(s);
            }
            // single loop with one bolt
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                // loop bolt 3  (also connect bolt3 to spout 1)
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt3");
                ret.add(new CycleDetectionScenario(String.format("(%d) One Loop", testNo), "Three level component hierarchy with 1 cycle in bolt3", tb.createTopology(), 1));
            }
            // single loop with three bolts
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                // loop bolt 3 -> 4 -> 5 -> 3 (also connect bolt3 to spout1)
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt5");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3");
                tb.setBolt("bolt5", new TestWordCounter(), 10).shuffleGrouping("bolt4");
                ret.add(new CycleDetectionScenario(String.format("(%d) One Loop", testNo), "Four level component hierarchy with 1 cycle in bolt3,bolt4,bolt5", tb.createTopology(), 1));
            }
            // two loops with three bolts, and one bolt
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("spout1");
                tb.setBolt("bolt11", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt12", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt21", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                tb.setBolt("bolt22", new TestWordCounter(), 10).shuffleGrouping("bolt2");
                // loop bolt 3 -> 4 -> 5 -> 3 (also connect bolt3 to spout1)
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt5");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3");
                tb.setBolt("bolt5", new TestWordCounter(), 10).shuffleGrouping("bolt4");
                // loop bolt 6  (also connect bolt6 to spout 1)
                tb.setBolt("bolt6", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt6");
                ret.add(new CycleDetectionScenario(String.format("(%d) Two Loops", testNo), "Four level component hierarchy with 2 cycles in bolt3,bolt4,bolt5 and bolt6", tb.createTopology(), 2));
            }
            // complex cycle
            {
                // (S1 -> B1 -> B2 -> B3 -> B4 <- S2), (B4 -> B3), (B4 -> B1)
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setSpout("spout2", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt4");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("bolt3").shuffleGrouping("spout2");
                ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#1", testNo), "Complex cycle (S1 -> B1 -> B2 -> B3 -> B4 <- S2), (B4 -> B3), (B4 -> B1)", tb.createTopology(), 1));
            }
            // another complex
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setSpout("spout1", new TestWordSpout(), 10);
                tb.setSpout("spout2", new TestWordSpout(), 10);
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("spout1").shuffleGrouping("bolt4").shuffleGrouping("bolt2");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4");
                tb.setBolt("bolt4", new TestWordCounter(), 10).shuffleGrouping("spout2");
                ret.add(new CycleDetectionScenario(String.format("(%d) Complex Loops#2", testNo), "Complex cycle 2 (S1 -> B1 <-> B2 -> B3 ), (S2 -> B4 -> B3), (B4 -> B1)", tb.createTopology(), 1));
            }
            // no spouts but with loops; the loops wont be detected
            {
                testNo++;
                tb = new TopologyBuilder();
                tb.setBolt("bolt1", new TestWordCounter(), 10).shuffleGrouping("bolt4").shuffleGrouping("bolt2");
                tb.setBolt("bolt2", new TestWordCounter(), 10).shuffleGrouping("bolt1");
                tb.setBolt("bolt3", new TestWordCounter(), 10).shuffleGrouping("bolt2").shuffleGrouping("bolt4");
                tb.setBolt("bolt4", new TestWordCounter(), 10);
                ret.add(new CycleDetectionScenario(String.format("(%d) No spout complex loops", testNo), "No Spouts, but with cycles (B1 <-> B2 -> B3 ), (B4 -> B3), (B4 -> B1)", tb.createTopology(), 0));
            }
            // now some randomly generated topologies
            int maxSpouts = 10;
            int maxBolts = 30;
            int randomTopoCnt = 100;
            for (int iRandTest = 0; iRandTest < randomTopoCnt; iRandTest++) {
                testNo++;
                tb = new TopologyBuilder();
                // topology component and connection counts
                int spoutCnt = ThreadLocalRandom.current().nextInt(0, maxSpouts) + 1;
                int boltCnt = ThreadLocalRandom.current().nextInt(0, maxBolts) + 1;
                int spoutToBoltConnectionCnt = ThreadLocalRandom.current().nextInt(spoutCnt * boltCnt) + 1;
                int boltToBoltConnectionCnt = ThreadLocalRandom.current().nextInt(boltCnt * boltCnt) + 1;
                Map<Integer, BoltDeclarer> boltDeclarers = new HashMap<>();
                for (int iSpout = 0; iSpout < spoutCnt; iSpout++) {
                    tb.setSpout("spout" + iSpout, new TestWordSpout(), 10);
                }
                for (int iBolt = 0; iBolt < boltCnt; iBolt++) {
                    boltDeclarers.put(iBolt, tb.setBolt("bolt" + iBolt, new TestWordCounter(), 10));
                }
                // spout to bolt connections
                for (int i = 0; i < spoutToBoltConnectionCnt; i++) {
                    int iSpout = ThreadLocalRandom.current().nextInt(0, spoutCnt);
                    int iBolt = ThreadLocalRandom.current().nextInt(0, boltCnt);
                    boltDeclarers.get(iBolt).shuffleGrouping("spout" + iSpout);
                }
                // bolt to bolt connections
                for (int i = 0; i < boltToBoltConnectionCnt; i++) {
                    int iBolt1 = ThreadLocalRandom.current().nextInt(0, boltCnt);
                    int iBolt2 = ThreadLocalRandom.current().nextInt(0, boltCnt);
                    boltDeclarers.get(iBolt2).shuffleGrouping("bolt" + iBolt1);
                }
                ret.add(new CycleDetectionScenario(String.format("(%d) Random Topo#%d", testNo, iRandTest), String.format("Random topology #%d, spouts=%d, bolts=%d, connections: fromSpouts=%d/fromBolts=%d", iRandTest, spoutCnt, boltCnt, spoutToBoltConnectionCnt, boltToBoltConnectionCnt), tb.createTopology(), -1));
            }
            return ret;
        }
    }
    List<String> testFailures = new ArrayList<>();
    new CycleDetectionScenario().createTestScenarios().forEach(x -> {
        LOG.info("==================== Running Test Scenario: {} =======================", x.testName);
        LOG.info("{}: {}", x.testName, x.testDescription);
        List<List<String>> loops = Utils.findComponentCycles(x.topology, x.testName);
        if (x.expectedCycles >= 0) {
            if (!loops.isEmpty()) {
                LOG.info("{} detected loops are \"{}\"", x.testName, loops.stream().map(y -> String.join(",", y)).collect(Collectors.joining(" ; ")));
            }
            if (loops.size() != x.expectedCycles) {
                testFailures.add(String.format("Test \"%s\" failed, detected cycles=%d does not match expected=%d for \"%s\"", x.testName, loops.size(), x.expectedCycles, x.testDescription));
                if (!loops.isEmpty()) {
                    testFailures.add(String.format("\t\tdetected loops are \"%s\"", loops.stream().map(y -> String.join(",", y)).collect(Collectors.joining(" ; "))));
                }
            }
        } else {
            // these are random topologies, with indeterminate number of loops
            LOG.info("{} detected loop count is \"{}\"", x.testName, loops.size());
        }
    });
    if (!testFailures.isEmpty()) {
        fail(String.join("\n", testFailures));
    }
}

Example 28 with BoltDeclarer

use of org.apache.storm.topology.BoltDeclarer in project storm by apache.

the class GenLoad method parseAndSubmit.

private static String parseAndSubmit(TopologyLoadConf tlc, String url) throws IOException, InvalidTopologyException, AuthorizationException, AlreadyAliveException {
    // First we need some configs
    Config conf = new Config();
    if (tlc.topoConf != null) {
        conf.putAll(tlc.topoConf);
    }
    // For some reason on the new code if ackers is null we get 0???
    Object ackers = conf.get(Config.TOPOLOGY_ACKER_EXECUTORS);
    Object workers = conf.get(Config.TOPOLOGY_WORKERS);
    if (ackers == null || ((Number) ackers).intValue() <= 0) {
        if (workers == null) {
            workers = 1;
        }
        conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, workers);
    }
    conf.registerMetricsConsumer(LoggingMetricsConsumer.class);
    conf.registerMetricsConsumer(HttpForwardingMetricsConsumer.class, url, 1);
    Map<String, String> workerMetrics = new HashMap<>();
    if (!NimbusClient.isLocalOverride()) {
        // sigar uses JNI and does not work in local mode
        workerMetrics.put("CPU", "org.apache.storm.metrics.sigar.CPUMetric");
    }
    conf.put(Config.TOPOLOGY_WORKER_METRICS, workerMetrics);
    conf.put(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS, 10);
    // Lets build a topology.
    TopologyBuilder builder = new TopologyBuilder();
    for (LoadCompConf spoutConf : tlc.spouts) {
        System.out.println("ADDING SPOUT " + spoutConf.id);
        SpoutDeclarer sd = builder.setSpout(spoutConf.id, new LoadSpout(spoutConf), spoutConf.parallelism);
        if (spoutConf.memoryLoad > 0) {
            sd.setMemoryLoad(spoutConf.memoryLoad);
        }
        if (spoutConf.cpuLoad > 0) {
            sd.setCPULoad(spoutConf.cpuLoad);
        }
    }
    Map<String, BoltDeclarer> boltDeclarers = new HashMap<>();
    Map<String, LoadBolt> bolts = new HashMap<>();
    if (tlc.bolts != null) {
        for (LoadCompConf boltConf : tlc.bolts) {
            System.out.println("ADDING BOLT " + boltConf.id);
            LoadBolt lb = new LoadBolt(boltConf);
            bolts.put(boltConf.id, lb);
            BoltDeclarer bd = builder.setBolt(boltConf.id, lb, boltConf.parallelism);
            if (boltConf.memoryLoad > 0) {
                bd.setMemoryLoad(boltConf.memoryLoad);
            }
            if (boltConf.cpuLoad > 0) {
                bd.setCPULoad(boltConf.cpuLoad);
            }
            boltDeclarers.put(boltConf.id, bd);
        }
    }
    if (tlc.streams != null) {
        for (InputStream in : tlc.streams) {
            BoltDeclarer declarer = boltDeclarers.get(in.toComponent);
            if (declarer == null) {
                throw new IllegalArgumentException("to bolt " + in.toComponent + " does not exist");
            }
            LoadBolt lb = bolts.get(in.toComponent);
            lb.add(in);
            in.groupingType.assign(declarer, in);
        }
    }
    String topoName = tlc.name + "-" + uniquifier++;
    StormSubmitter.submitTopology(topoName, conf, builder.createTopology());
    return topoName;
}

Example 29 with BoltDeclarer

use of org.apache.storm.topology.BoltDeclarer in project storm by apache.

the class TestUtilsForBlacklistScheduler method buildTopology.

public static StormTopology buildTopology(int numSpout, int numBolt, int spoutParallelism, int boltParallelism) {
    LOG.debug("buildTopology with -> numSpout: " + numSpout + " spoutParallelism: " + spoutParallelism + " numBolt: " + numBolt + " boltParallelism: " + boltParallelism);
    TopologyBuilder builder = new TopologyBuilder();
    for (int i = 0; i < numSpout; i++) {
        SpoutDeclarer s1 = builder.setSpout("spout-" + i, new TestSpout(), spoutParallelism);
    }
    int j = 0;
    for (int i = 0; i < numBolt; i++) {
        if (j >= numSpout) {
            j = 0;
        }
        BoltDeclarer b1 = builder.setBolt("bolt-" + i, new TestBolt(), boltParallelism).shuffleGrouping("spout-" + j);
    }
    return builder.createTopology();
}

Example 30 with BoltDeclarer

use of org.apache.storm.topology.BoltDeclarer in project storm by apache.

the class ConstSpoutNullBoltTopo method getTopology.

static StormTopology getTopology(Map<String, Object> conf) {
    // 1 -  Setup Spout   --------
    ConstSpout spout = new ConstSpout("some data").withOutputFields("str");
    // 2 -  Setup DevNull Bolt   --------
    DevNullBolt bolt = new DevNullBolt();
    // 3 - Setup Topology  --------
    TopologyBuilder builder = new TopologyBuilder();
    int numSpouts = Helper.getInt(conf, SPOUT_COUNT, 1);
    builder.setSpout(SPOUT_ID, spout, numSpouts);
    int numBolts = Helper.getInt(conf, BOLT_COUNT, 1);
    BoltDeclarer bd = builder.setBolt(BOLT_ID, bolt, numBolts);
    System.err.printf("====> Using : numSpouts = %d , numBolts = %d\n", numSpouts, numBolts);
    String groupingType = Helper.getStr(conf, GROUPING);
    if (groupingType == null || groupingType.equalsIgnoreCase(DEFAULT_GROUPING)) {
        bd.localOrShuffleGrouping(SPOUT_ID);
    } else if (groupingType.equalsIgnoreCase(SHUFFLE_GROUPING)) {
        bd.shuffleGrouping(SPOUT_ID);
    }
    return builder.createTopology();
}