Examples with LocalCluster org.apache.storm.LocalCluster used on opensource projects

Example 21 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class SingleJoinExample method main.

public static void main(String[] args) throws Exception {
    FeederSpout genderSpout = new FeederSpout(new Fields("id", "gender"));
    FeederSpout ageSpout = new FeederSpout(new Fields("id", "age"));
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("gender", genderSpout);
    builder.setSpout("age", ageSpout);
    builder.setBolt("join", new SingleJoinBolt(new Fields("gender", "age"))).fieldsGrouping("gender", new Fields("id")).fieldsGrouping("age", new Fields("id"));
    Config conf = new Config();
    conf.setDebug(true);
    try (LocalCluster cluster = new LocalCluster();
        LocalTopology topo = cluster.submitTopology("join-example", conf, builder.createTopology())) {
        for (int i = 0; i < 10; i++) {
            String gender;
            if (i % 2 == 0) {
                gender = "male";
            } else {
                gender = "female";
            }
            genderSpout.feed(new Values(i, gender));
        }
        for (int i = 9; i >= 0; i--) {
            ageSpout.feed(new Values(i, i + 20));
        }
        Utils.sleep(2000);
    }
}

Example 22 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class Flux method runCli.

private static void runCli(CommandLine cmd) throws Exception {
    if (!cmd.hasOption(OPTION_NO_SPLASH)) {
        printSplash();
    }
    boolean dumpYaml = cmd.hasOption("dump-yaml");
    TopologyDef topologyDef = null;
    String filePath = (String) cmd.getArgList().get(0);
    // TODO conditionally load properties from a file our resource
    String filterProps = null;
    if (cmd.hasOption(OPTION_FILTER)) {
        filterProps = cmd.getOptionValue(OPTION_FILTER);
    }
    boolean envFilter = cmd.hasOption(OPTION_ENV_FILTER);
    if (cmd.hasOption(OPTION_RESOURCE)) {
        printf("Parsing classpath resource: %s", filePath);
        topologyDef = FluxParser.parseResource(filePath, dumpYaml, true, filterProps, envFilter);
    } else {
        printf("Parsing file: %s", new File(filePath).getAbsolutePath());
        topologyDef = FluxParser.parseFile(filePath, dumpYaml, true, filterProps, envFilter);
    }
    String topologyName = topologyDef.getName();
    // merge contents of `config` into topology config
    Config conf = FluxBuilder.buildConfig(topologyDef);
    ExecutionContext context = new ExecutionContext(topologyDef, conf);
    StormTopology topology = FluxBuilder.buildTopology(context);
    if (!cmd.hasOption(OPTION_NO_DETAIL)) {
        printTopologyInfo(context);
    }
    if (!cmd.hasOption(OPTION_DRY_RUN)) {
        if (cmd.hasOption(OPTION_REMOTE)) {
            LOG.info("Running remotely...");
            // should the topology be active or inactive
            SubmitOptions submitOptions = null;
            if (cmd.hasOption(OPTION_INACTIVE)) {
                LOG.info("Deploying topology in an INACTIVE state...");
                submitOptions = new SubmitOptions(TopologyInitialStatus.INACTIVE);
            } else {
                LOG.info("Deploying topology in an ACTIVE state...");
                submitOptions = new SubmitOptions(TopologyInitialStatus.ACTIVE);
            }
            StormSubmitter.submitTopology(topologyName, conf, topology, submitOptions, null);
        } else {
            LOG.info("Running in local mode...");
            String sleepStr = cmd.getOptionValue(OPTION_SLEEP);
            Long sleepTime = DEFAULT_LOCAL_SLEEP_TIME;
            if (sleepStr != null) {
                sleepTime = Long.parseLong(sleepStr);
            }
            LOG.debug("Sleep time: {}", sleepTime);
            LocalCluster cluster = null;
            // in-process or external zookeeper
            if (cmd.hasOption(OPTION_ZOOKEEPER)) {
                String zkStr = cmd.getOptionValue(OPTION_ZOOKEEPER);
                LOG.info("Using ZooKeeper at '{}' instead of in-process one.", zkStr);
                long zkPort = DEFAULT_ZK_PORT;
                String zkHost = null;
                if (zkStr.contains(":")) {
                    String[] hostPort = zkStr.split(":");
                    zkHost = hostPort[0];
                    zkPort = hostPort.length > 1 ? Long.parseLong(hostPort[1]) : DEFAULT_ZK_PORT;
                } else {
                    zkHost = zkStr;
                }
                // the following constructor is only available in 0.9.3 and later
                try {
                    cluster = new LocalCluster(zkHost, zkPort);
                } catch (NoSuchMethodError e) {
                    LOG.error("The --zookeeper option can only be used with Apache Storm 0.9.3 and later.");
                    System.exit(1);
                }
            } else {
                cluster = new LocalCluster();
            }
            try (LocalTopology topo = cluster.submitTopology(topologyName, conf, topology)) {
                Utils.sleep(sleepTime);
            } finally {
                cluster.shutdown();
            }
        }
    }
}

Example 23 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class JoinExample method main.

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    // a stream of (number, square) pairs
    PairStream<Integer, Integer> squares = builder.newStream(new NumberSpout(x -> x * x), new PairValueMapper<>(0, 1));
    // a stream of (number, cube) pairs
    PairStream<Integer, Integer> cubes = builder.newStream(new NumberSpout(x -> x * x * x), new PairValueMapper<>(0, 1));
    // create a windowed stream of five seconds duration
    squares.window(TumblingWindows.of(Duration.seconds(5))).join(cubes).print();
    Config config = new Config();
    if (args.length > 0) {
        config.setNumWorkers(1);
        StormSubmitter.submitTopologyWithProgressBar(args[0], config, builder.build());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalCluster.LocalTopology topo = cluster.submitTopology("test", config, builder.build())) {
            Utils.sleep(60_000);
        }
    }
}

Example 24 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class StateQueryExample method main.

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    StreamState<String, Long> ss = builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2).mapToPair(w -> Pair.of(w, 1)).updateStateByKey(0L, (count, val) -> count + 1);
    /*
         * A stream of words emitted by the QuerySpout is used as
         * the keys to query the state.
         */
    builder.newStream(new QuerySpout(), new ValueMapper<String>(0)).stateQuery(ss).print();
    Config config = new Config();
    // use redis based state store for persistence
    config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
    if (args.length > 0) {
        config.setNumWorkers(1);
        StormSubmitter.submitTopologyWithProgressBar(args[0], config, builder.build());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalCluster.LocalTopology topo = cluster.submitTopology("test", config, builder.build())) {
            Utils.sleep(60_000);
        }
    }
}

Example 25 with LocalCluster

use of org.apache.storm.LocalCluster in project storm by apache.

the class StatefulWordCount method main.

public static void main(String[] args) throws Exception {
    StreamBuilder builder = new StreamBuilder();
    // a stream of words
    builder.newStream(new TestWordSpout(), new ValueMapper<String>(0), 2).window(TumblingWindows.of(BaseWindowedBolt.Duration.seconds(2))).mapToPair(w -> Pair.of(w, 1)).countByKey().updateStateByKey(0L, (state, count) -> state + count).toPairStream().print();
    Config config = new Config();
    // use redis based state store for persistence
    config.put(Config.TOPOLOGY_STATE_PROVIDER, "org.apache.storm.redis.state.RedisKeyValueStateProvider");
    if (args.length > 0) {
        config.setNumWorkers(1);
        StormSubmitter.submitTopologyWithProgressBar(args[0], config, builder.build());
    } else {
        try (LocalCluster cluster = new LocalCluster();
            LocalCluster.LocalTopology topo = cluster.submitTopology("test", config, builder.build())) {
            Utils.sleep(60_000);
        }
    }
}