Examples with LocalCluster org.apache.storm.LocalCluster used on opensource projects

Example 46 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class MultipleLoggerTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");
    Config conf = new Config();
    conf.setDebug(true);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(2);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology())) {
            Utils.sleep(10000);
        }
    }
}

Example 47 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class ResourceAwareExampleTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    SpoutDeclarer spout = builder.setSpout("word", new TestWordSpout(), 5);
    //set cpu requirement
    spout.setCPULoad(20);
    //set onheap and offheap memory requirement
    spout.setMemoryLoad(64, 16);
    BoltDeclarer bolt1 = builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    //sets cpu requirement.  Not neccessary to set both CPU and memory.
    //For requirements not set, a default value will be used
    bolt1.setCPULoad(15);
    BoltDeclarer bolt2 = builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    bolt2.setMemoryLoad(100);
    Config conf = new Config();
    conf.setDebug(true);
    /**
     * Use to limit the maximum amount of memory (in MB) allocated to one worker process.
     * Can be used to spread executors to to multiple workers
     */
    conf.setTopologyWorkerMaxHeapSize(1024.0);
    //topology priority describing the importance of the topology in decreasing importance starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases).
    //Recommended range of 0-29 but no hard limit set.
    conf.setTopologyPriority(29);
    // Set strategy to schedule topology. If not specified, default to org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy
    conf.setTopologyStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class);
    if (args != null && args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("test", conf, builder.createTopology())) {
            Utils.sleep(10000);
        }
    }
}

Example 48 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class ExampleJmsTopology method main.

@SuppressWarnings("serial")
public static void main(String[] args) throws Exception {
    // JMS Queue Provider
    JmsProvider jmsQueueProvider = new SpringJmsProvider("jms-activemq.xml", "jmsConnectionFactory", "notificationQueue");
    // JMS Topic provider
    JmsProvider jmsTopicProvider = new SpringJmsProvider("jms-activemq.xml", "jmsConnectionFactory", "notificationTopic");
    // JMS Producer
    JmsTupleProducer producer = new JsonTupleProducer();
    // JMS Queue Spout
    JmsSpout queueSpout = new JmsSpout();
    queueSpout.setJmsProvider(jmsQueueProvider);
    queueSpout.setJmsTupleProducer(producer);
    queueSpout.setJmsAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
    // allow multiple instances
    queueSpout.setDistributed(true);
    TopologyBuilder builder = new TopologyBuilder();
    // spout with 5 parallel instances
    builder.setSpout(JMS_QUEUE_SPOUT, queueSpout, 5);
    // intermediate bolt, subscribes to jms spout, anchors on tuples, and auto-acks
    builder.setBolt(INTERMEDIATE_BOLT, new GenericBolt("INTERMEDIATE_BOLT", true, true, new Fields("json")), 3).shuffleGrouping(JMS_QUEUE_SPOUT);
    // bolt that subscribes to the intermediate bolt, and auto-acks
    // messages.
    builder.setBolt(FINAL_BOLT, new GenericBolt("FINAL_BOLT", true, true), 3).shuffleGrouping(INTERMEDIATE_BOLT);
    // bolt that subscribes to the intermediate bolt, and publishes to a JMS Topic
    JmsBolt jmsBolt = new JmsBolt();
    jmsBolt.setJmsProvider(jmsTopicProvider);
    // anonymous message producer just calls toString() on the tuple to create a jms message
    jmsBolt.setJmsMessageProducer(new JmsMessageProducer() {

        @Override
        public Message toMessage(Session session, ITuple input) throws JMSException {
            System.out.println("Sending JMS Message:" + input.toString());
            TextMessage tm = session.createTextMessage(input.toString());
            return tm;
        }
    });
    builder.setBolt(JMS_TOPIC_BOLT, jmsBolt).shuffleGrouping(INTERMEDIATE_BOLT);
    // JMS Topic spout
    JmsSpout topicSpout = new JmsSpout();
    topicSpout.setJmsProvider(jmsTopicProvider);
    topicSpout.setJmsTupleProducer(producer);
    topicSpout.setJmsAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
    topicSpout.setDistributed(false);
    builder.setSpout(JMS_TOPIC_SPOUT, topicSpout);
    builder.setBolt(ANOTHER_BOLT, new GenericBolt("ANOTHER_BOLT", true, true), 1).shuffleGrouping(JMS_TOPIC_SPOUT);
    Config conf = new Config();
    if (args.length > 0) {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
    } else {
        conf.setDebug(true);
        try (LocalCluster cluster = new LocalCluster();
            LocalTopology topo = cluster.submitTopology("storm-jms-example", conf, builder.createTopology())) {
            Utils.sleep(60000);
        }
    }
}

Example 49 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class StormMqttIntegrationTest method testMqttTopology.

@Test
public void testMqttTopology() throws Exception {
    MQTT client = new MQTT();
    client.setTracer(new MqttLogger());
    URI uri = URI.create("tcp://localhost:1883");
    client.setHost(uri);
    client.setClientId("MQTTSubscriber");
    client.setCleanSession(false);
    BlockingConnection connection = client.blockingConnection();
    connection.connect();
    Topic[] topics = { new Topic("/integration-result", QoS.AT_LEAST_ONCE) };
    byte[] qoses = connection.subscribe(topics);
    try (LocalCluster cluster = new LocalCluster();
        LocalTopology topo = cluster.submitTopology("test", new Config(), buildMqttTopology())) {
        LOG.info("topology started");
        while (!spoutActivated) {
            Thread.sleep(500);
        }
        // publish a retained message to the broker
        MqttOptions options = new MqttOptions();
        options.setCleanConnection(false);
        MqttPublisher publisher = new MqttPublisher(options, true);
        publisher.connectMqtt("MqttPublisher");
        publisher.publish(new MqttMessage(TEST_TOPIC, "test".getBytes()));
        LOG.info("published message");
        Message message = connection.receive();
        LOG.info("Message recieved on topic: {}", message.getTopic());
        LOG.info("Payload: {}", new String(message.getPayload()));
        message.ack();
        Assert.assertArrayEquals(message.getPayload(), RESULT_PAYLOAD.getBytes());
        Assert.assertEquals(message.getTopic(), RESULT_TOPIC);
    }
}

Example 50 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class ConstSpoutNullBoltTopo method main.

/**
     * ConstSpout -> DevNullBolt with configurable grouping (default localOrShuffle)
     */
public static void main(String[] args) throws Exception {
    if (args.length <= 0) {
        // For IDE based profiling ... submit topology to local cluster
        Config conf = new Config();
        final LocalCluster cluster = Helper.runOnLocalCluster(TOPOLOGY_NAME, getTopology(conf));
        Helper.setupShutdownHook(cluster, TOPOLOGY_NAME);
        while (true) {
            //  run indefinitely till Ctrl-C
            Thread.sleep(20_000_000);
        }
    } else {
        // For measuring perf against a Storm cluster
        if (args.length > 2) {
            System.err.println("args: runDurationSec  [optionalConfFile]");
            return;
        }
        Integer durationSec = Integer.parseInt(args[0]);
        Map topoConf = (args.length == 2) ? Utils.findAndReadConfigFile(args[1]) : new Config();
        //  Submit topology to storm cluster
        Helper.runOnClusterAndPrintMetrics(durationSec, TOPOLOGY_NAME, topoConf, getTopology(topoConf));
    }
}