Examples with StormTopology org.apache.storm.generated.StormTopology used on opensource projects

Example 76 with StormTopology

use of org.apache.storm.generated.StormTopology in project open-kilda by telstra.

the class StatsTopologyTest method setupOnce.

@BeforeClass
public static void setupOnce() throws Exception {
    Properties configOverlay = getZooKeeperProperties(STATS_TOPOLOGY_TEST_ZOOKEEPER_PORT, ROOT_NODE);
    configOverlay.putAll(getKafkaProperties(STATS_TOPOLOGY_TEST_KAFKA_PORT));
    configOverlay.setProperty("opentsdb.metric.prefix", METRIC_PREFIX);
    AbstractStormTest.startZooKafka(configOverlay);
    setStartSignal(STATS_TOPOLOGY_TEST_ZOOKEEPER_PORT, ROOT_NODE, COMPONENT_NAME, RUN_ID);
    AbstractStormTest.startStorm(COMPONENT_NAME, RUN_ID);
    LaunchEnvironment launchEnvironment = makeLaunchEnvironment();
    launchEnvironment.setupOverlay(configOverlay);
    persistenceManager = InMemoryGraphPersistenceManager.newInstance();
    persistenceManager.install();
    StatsTopology statsTopology = new StatsTopology(launchEnvironment, persistenceManager);
    statsTopologyConfig = statsTopology.getConfig();
    StormTopology stormTopology = statsTopology.createTopology();
    Config config = stormConfig();
    cluster.submitTopology(StatsTopologyTest.class.getSimpleName(), config, stormTopology);
    otsdbConsumer = new TestKafkaConsumer(statsTopologyConfig.getKafkaOtsdbTopic(), kafkaConsumerProperties(UUID.randomUUID().toString(), COMPONENT_NAME, RUN_ID));
    otsdbConsumer.start();
    flowRepository = persistenceManager.getRepositoryFactory().createFlowRepository();
    switchRepository = persistenceManager.getRepositoryFactory().createSwitchRepository();
    sleep(TOPOLOGY_START_TIMEOUT);
}

Example 77 with StormTopology

use of org.apache.storm.generated.StormTopology in project pancm_project by xuwujing.

the class WordCountApp method main.

/**
 * The entry point of application.
 *
 * @param args the input arguments
 * @throws InterruptedException     the interrupted exception
 * @throws AlreadyAliveException    the already alive exception
 * @throws InvalidTopologyException the invalid topology exception
 */
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
    // 定义拓扑
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word-reader", new WordReader());
    builder.setBolt("word-normalizer", new WordNormalizer()).shuffleGrouping("word-reader");
    builder.setBolt("word-counter", new WordCounter()).fieldsGrouping("word-normalizer", new Fields("word"));
    StormTopology topology = builder.createTopology();
    // 配置
    Config conf = new Config();
    String fileName = "words.txt";
    conf.put("fileName", fileName);
    conf.setDebug(false);
    // 运行拓扑
    System.out.println("开始...");
    if (args != null && args.length > 0) {
        // 有参数时，表示向集群提交作业，并把第一个参数当做topology名称
        System.out.println("远程模式");
        try {
            StormSubmitter.submitTopology(args[0], conf, topology);
        } catch (AuthorizationException e) {
            e.printStackTrace();
        }
    } else {
        // 没有参数时，本地提交
        // 启动本地模式
        System.out.println("本地模式");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("Getting-Started-Topologie", conf, topology);
        Thread.sleep(5000);
        // 关闭本地集群
        cluster.shutdown();
    }
    System.out.println("结束");
}

Example 78 with StormTopology

use of org.apache.storm.generated.StormTopology in project pancm_project by xuwujing.

the class WordCountApp method main.

/**
 * The entry point of application.
 *
 * @param args the input arguments
 * @throws InterruptedException     the interrupted exception
 * @throws AlreadyAliveException    the already alive exception
 * @throws InvalidTopologyException the invalid topology exception
 */
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
    // 定义拓扑
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word-reader", new WordReader());
    builder.setBolt("word-normalizer", new WordNormalizer()).shuffleGrouping("word-reader");
    builder.setBolt("word-counter", new WordCounter()).fieldsGrouping("word-normalizer", new Fields("word"));
    StormTopology topology = builder.createTopology();
    // 配置
    Config conf = new Config();
    String fileName = "words.txt";
    conf.put("fileName", fileName);
    conf.setDebug(false);
    // 运行拓扑
    System.out.println("开始...");
    if (args != null && args.length > 0) {
        // 有参数时，表示向集群提交作业，并把第一个参数当做topology名称
        System.out.println("远程模式");
        try {
            StormSubmitter.submitTopology(args[0], conf, topology);
        } catch (AuthorizationException e) {
            e.printStackTrace();
        }
    } else {
        // 没有参数时，本地提交
        // 启动本地模式
        System.out.println("本地模式");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology("Getting-Started-Topologie", conf, topology);
        Thread.sleep(5000);
        // 关闭本地集群
        cluster.shutdown();
    }
    System.out.println("结束");
}

Example 79 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class TopologyDetails method getComponents.

/**
     * Returns a representation of the non-system components of the topology graph
     * Each Component object in the returning map is populated with the list of its
     * parents, children and execs assigned to that component.
     * @return a map of components
     */
public Map<String, Component> getComponents() {
    Map<String, Component> all_comp = new HashMap<>();
    StormTopology storm_topo = this.topology;
    // spouts
    if (storm_topo.get_spouts() != null) {
        for (Map.Entry<String, SpoutSpec> spoutEntry : storm_topo.get_spouts().entrySet()) {
            if (!Utils.isSystemId(spoutEntry.getKey())) {
                Component newComp;
                if (all_comp.containsKey(spoutEntry.getKey())) {
                    newComp = all_comp.get(spoutEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                } else {
                    newComp = new Component(spoutEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                    all_comp.put(spoutEntry.getKey(), newComp);
                }
                newComp.type = Component.ComponentType.SPOUT;
                for (Map.Entry<GlobalStreamId, Grouping> spoutInput : spoutEntry.getValue().get_common().get_inputs().entrySet()) {
                    newComp.parents.add(spoutInput.getKey().get_componentId());
                    if (!all_comp.containsKey(spoutInput.getKey().get_componentId())) {
                        all_comp.put(spoutInput.getKey().get_componentId(), new Component(spoutInput.getKey().get_componentId()));
                    }
                    all_comp.get(spoutInput.getKey().get_componentId()).children.add(spoutEntry.getKey());
                }
            }
        }
    }
    // bolts
    if (storm_topo.get_bolts() != null) {
        for (Map.Entry<String, Bolt> boltEntry : storm_topo.get_bolts().entrySet()) {
            if (!Utils.isSystemId(boltEntry.getKey())) {
                Component newComp;
                if (all_comp.containsKey(boltEntry.getKey())) {
                    newComp = all_comp.get(boltEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                } else {
                    newComp = new Component(boltEntry.getKey());
                    newComp.execs = componentToExecs(newComp.id);
                    all_comp.put(boltEntry.getKey(), newComp);
                }
                newComp.type = Component.ComponentType.BOLT;
                for (Map.Entry<GlobalStreamId, Grouping> boltInput : boltEntry.getValue().get_common().get_inputs().entrySet()) {
                    newComp.parents.add(boltInput.getKey().get_componentId());
                    if (!all_comp.containsKey(boltInput.getKey().get_componentId())) {
                        all_comp.put(boltInput.getKey().get_componentId(), new Component(boltInput.getKey().get_componentId()));
                    }
                    all_comp.get(boltInput.getKey().get_componentId()).children.add(boltEntry.getKey());
                }
            }
        }
    }
    return all_comp;
}

Example 80 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class Nimbus method computeExecutors.

private List<List<Integer>> computeExecutors(String topoId, StormBase base) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
    BlobStore store = blobStore;
    assert (base != null);
    Map<String, Integer> compToExecutors = base.get_component_executors();
    Map<String, Object> topoConf = readTopoConfAsNimbus(topoId, store);
    StormTopology topology = readStormTopologyAsNimbus(topoId, store);
    List<List<Integer>> ret = new ArrayList<>();
    if (compToExecutors != null) {
        Map<Integer, String> taskInfo = StormCommon.stormTaskInfo(topology, topoConf);
        Map<String, List<Integer>> compToTaskList = Utils.reverseMap(taskInfo);
        for (Entry<String, List<Integer>> entry : compToTaskList.entrySet()) {
            List<Integer> comps = entry.getValue();
            comps.sort(null);
            Integer numExecutors = compToExecutors.get(entry.getKey());
            if (numExecutors != null) {
                List<List<Integer>> partitioned = Utils.partitionFixed(numExecutors, comps);
                for (List<Integer> partition : partitioned) {
                    ret.add(Arrays.asList(partition.get(0), partition.get(partition.size() - 1)));
                }
            }
        }
    }
    return ret;
}