Examples with StormTopology org.apache.storm.generated.StormTopology used on opensource projects

Example 81 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class Nimbus method startTopology.

private void startTopology(String topoName, String topoId, TopologyStatus initStatus) throws KeyNotFoundException, AuthorizationException, IOException, InvalidTopologyException {
    assert (TopologyStatus.ACTIVE == initStatus || TopologyStatus.INACTIVE == initStatus);
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Map<String, Object> topoConf = readTopoConf(topoId, store);
    StormTopology topology = StormCommon.systemTopology(topoConf, readStormTopology(topoId, store));
    Map<String, Integer> numExecutors = new HashMap<>();
    for (Entry<String, Object> entry : StormCommon.allComponents(topology).entrySet()) {
        numExecutors.put(entry.getKey(), StormCommon.numStartExecutors(entry.getValue()));
    }
    LOG.info("Activating {}: {}", topoName, topoId);
    StormBase base = new StormBase();
    base.set_name(topoName);
    base.set_launch_time_secs(Time.currentTimeSecs());
    base.set_status(initStatus);
    base.set_num_workers(Utils.getInt(topoConf.get(Config.TOPOLOGY_WORKERS), 0));
    base.set_component_executors(numExecutors);
    base.set_owner((String) topoConf.get(Config.TOPOLOGY_SUBMITTER_USER));
    base.set_component_debug(new HashMap<>());
    state.activateStorm(topoId, base);
    notifyTopologyActionListener(topoName, "activate");
}

Example 82 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class StormCommon method stormTaskInfoImpl.

/*
     * Returns map from task -> componentId
     */
protected Map<Integer, String> stormTaskInfoImpl(StormTopology userTopology, Map stormConf) throws InvalidTopologyException {
    Map<Integer, String> taskIdToComponentId = new HashMap<>();
    StormTopology systemTopology = systemTopology(stormConf, userTopology);
    Map<String, Object> components = allComponents(systemTopology);
    Map<String, Integer> componentIdToTaskNum = new TreeMap<>();
    for (Map.Entry<String, Object> entry : components.entrySet()) {
        Map conf = componentConf(entry.getValue());
        Object taskNum = conf.get(Config.TOPOLOGY_TASKS);
        componentIdToTaskNum.put(entry.getKey(), Utils.getInt(taskNum));
    }
    int taskId = 1;
    for (Map.Entry<String, Integer> entry : componentIdToTaskNum.entrySet()) {
        String componentId = entry.getKey();
        Integer taskNum = entry.getValue();
        while (taskNum > 0) {
            taskIdToComponentId.put(taskId, componentId);
            taskNum--;
            taskId++;
        }
    }
    return taskIdToComponentId;
}

Example 83 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class StormCommon method systemTopologyImpl.

protected StormTopology systemTopologyImpl(Map stormConf, StormTopology topology) throws InvalidTopologyException {
    validateBasic(topology);
    StormTopology ret = topology.deepCopy();
    addAcker(stormConf, ret);
    if (hasEventLoggers(stormConf)) {
        addEventLogger(stormConf, ret);
    }
    addMetricComponents(stormConf, ret);
    addSystemComponents(stormConf, ret);
    addMetricStreams(ret);
    addSystemStreams(ret);
    validateStructure(ret);
    return ret;
}

Example 84 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class TestDefaultResourceAwareStrategy method testDefaultResourceAwareStrategy.

/**
     * test if the scheduling logic for the DefaultResourceAwareStrategy is correct
     */
@Test
public void testDefaultResourceAwareStrategy() {
    int spoutParallelism = 1;
    int boltParallelism = 2;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new TestUtilsForResourceAwareScheduler.TestSpout(), spoutParallelism);
    builder.setBolt("bolt-1", new TestUtilsForResourceAwareScheduler.TestBolt(), boltParallelism).shuffleGrouping("spout");
    builder.setBolt("bolt-2", new TestUtilsForResourceAwareScheduler.TestBolt(), boltParallelism).shuffleGrouping("bolt-1");
    builder.setBolt("bolt-3", new TestUtilsForResourceAwareScheduler.TestBolt(), boltParallelism).shuffleGrouping("bolt-2");
    StormTopology stormToplogy = builder.createTopology();
    Config conf = new Config();
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    Map<String, Number> resourceMap = new HashMap<String, Number>();
    resourceMap.put(Config.SUPERVISOR_CPU_CAPACITY, 150.0);
    resourceMap.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1500.0);
    Map<String, SupervisorDetails> supMap = TestUtilsForResourceAwareScheduler.genSupervisors(4, 4, resourceMap);
    conf.putAll(Utils.readDefaultConfig());
    conf.put(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY, org.apache.storm.scheduler.resource.strategies.eviction.DefaultEvictionStrategy.class.getName());
    conf.put(Config.RESOURCE_AWARE_SCHEDULER_PRIORITY_STRATEGY, org.apache.storm.scheduler.resource.strategies.priority.DefaultSchedulingPriorityStrategy.class.getName());
    conf.put(Config.TOPOLOGY_SCHEDULER_STRATEGY, org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy.class.getName());
    conf.put(Config.TOPOLOGY_COMPONENT_CPU_PCORE_PERCENT, 50.0);
    conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_OFFHEAP_MEMORY_MB, 250);
    conf.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 250);
    conf.put(Config.TOPOLOGY_PRIORITY, 0);
    conf.put(Config.TOPOLOGY_NAME, "testTopology");
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, TestUtilsForResourceAwareScheduler.genExecsAndComps(stormToplogy), this.currentTime);
    Map<String, TopologyDetails> topoMap = new HashMap<String, TopologyDetails>();
    topoMap.put(topo.getId(), topo);
    Topologies topologies = new Topologies(topoMap);
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), conf);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(conf);
    rs.schedule(topologies, cluster);
    Map<String, List<String>> nodeToComps = new HashMap<String, List<String>>();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : cluster.getAssignments().get("testTopology-id").getExecutorToSlot().entrySet()) {
        WorkerSlot ws = entry.getValue();
        ExecutorDetails exec = entry.getKey();
        if (!nodeToComps.containsKey(ws.getNodeId())) {
            nodeToComps.put(ws.getNodeId(), new LinkedList<String>());
        }
        nodeToComps.get(ws.getNodeId()).add(topo.getExecutorToComponent().get(exec));
    }
    /**
         * check for correct scheduling
         * Since all the resource availabilites on nodes are the same in the beginining
         * DefaultResourceAwareStrategy can arbitrarily pick one thus we must find if a particular scheduling
         * exists on a node the the cluster.
         */
    //one node should have the below scheduling
    List<String> node1 = new LinkedList<>();
    node1.add("spout");
    node1.add("bolt-1");
    node1.add("bolt-2");
    Assert.assertTrue("Check DefaultResourceAwareStrategy scheduling", checkDefaultStrategyScheduling(nodeToComps, node1));
    //one node should have the below scheduling
    List<String> node2 = new LinkedList<>();
    node2.add("bolt-3");
    node2.add("bolt-1");
    node2.add("bolt-2");
    Assert.assertTrue("Check DefaultResourceAwareStrategy scheduling", checkDefaultStrategyScheduling(nodeToComps, node2));
    //one node should have the below scheduling
    List<String> node3 = new LinkedList<>();
    node3.add("bolt-3");
    Assert.assertTrue("Check DefaultResourceAwareStrategy scheduling", checkDefaultStrategyScheduling(nodeToComps, node3));
    //three used and one node should be empty
    Assert.assertEquals("only three nodes should be used", 3, nodeToComps.size());
}

Example 85 with StormTopology

use of org.apache.storm.generated.StormTopology in project storm by apache.

the class TestResourceAwareScheduler method testHeterogeneousCluster.

@Test
public void testHeterogeneousCluster() {
    INimbus iNimbus = new TestUtilsForResourceAwareScheduler.INimbusTest();
    // strong supervisor node
    Map<String, Number> resourceMap1 = new HashMap<>();
    resourceMap1.put(Config.SUPERVISOR_CPU_CAPACITY, 800.0);
    resourceMap1.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 4096.0);
    // weak supervisor node
    Map<String, Number> resourceMap2 = new HashMap<>();
    resourceMap2.put(Config.SUPERVISOR_CPU_CAPACITY, 200.0);
    resourceMap2.put(Config.SUPERVISOR_MEMORY_CAPACITY_MB, 1024.0);
    Map<String, SupervisorDetails> supMap = new HashMap<String, SupervisorDetails>();
    for (int i = 0; i < 2; i++) {
        List<Number> ports = new LinkedList<Number>();
        for (int j = 0; j < 4; j++) {
            ports.add(j);
        }
        SupervisorDetails sup = new SupervisorDetails("sup-" + i, "host-" + i, null, ports, (Map) (i == 0 ? resourceMap1 : resourceMap2));
        supMap.put(sup.getId(), sup);
    }
    // topo1 has one single huge task that can not be handled by the small-super
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 1).setCPULoad(300.0).setMemoryLoad(2000.0, 48.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 1, executorMap1, 0);
    // topo2 has 4 large tasks
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpout2", new TestWordSpout(), 4).setCPULoad(100.0).setMemoryLoad(500.0, 12.0);
    StormTopology stormTopology2 = builder2.createTopology();
    Config config2 = new Config();
    config2.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap2 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 1, executorMap2, 0);
    // topo3 has 4 large tasks
    TopologyBuilder builder3 = new TopologyBuilder();
    builder3.setSpout("wordSpout3", new TestWordSpout(), 4).setCPULoad(20.0).setMemoryLoad(200.0, 56.0);
    StormTopology stormTopology3 = builder3.createTopology();
    Config config3 = new Config();
    config3.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap3 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology3);
    TopologyDetails topology3 = new TopologyDetails("topology3", config2, stormTopology3, 1, executorMap3, 0);
    // topo4 has 12 small tasks, whose mem usage does not exactly divide a node's mem capacity
    TopologyBuilder builder4 = new TopologyBuilder();
    builder4.setSpout("wordSpout4", new TestWordSpout(), 12).setCPULoad(30.0).setMemoryLoad(100.0, 0.0);
    StormTopology stormTopology4 = builder4.createTopology();
    Config config4 = new Config();
    config4.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap4 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology4);
    TopologyDetails topology4 = new TopologyDetails("topology4", config4, stormTopology4, 1, executorMap4, 0);
    // topo5 has 40 small tasks, it should be able to exactly use up both the cpu and mem in the cluster
    TopologyBuilder builder5 = new TopologyBuilder();
    builder5.setSpout("wordSpout5", new TestWordSpout(), 40).setCPULoad(25.0).setMemoryLoad(100.0, 28.0);
    StormTopology stormTopology5 = builder5.createTopology();
    Config config5 = new Config();
    config5.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap5 = TestUtilsForResourceAwareScheduler.genExecsAndComps(stormTopology5);
    TopologyDetails topology5 = new TopologyDetails("topology5", config5, stormTopology5, 1, executorMap5, 0);
    // Test1: Launch topo 1-3 together, it should be able to use up either mem or cpu resource due to exact division
    Cluster cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config1);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    Map<String, TopologyDetails> topoMap = new HashMap<>();
    topoMap.put(topology1.getId(), topology1);
    topoMap.put(topology2.getId(), topology2);
    topoMap.put(topology3.getId(), topology3);
    Topologies topologies = new Topologies(topoMap);
    rs.prepare(config1);
    rs.schedule(topologies, cluster);
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology1.getId()));
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology2.getId()));
    Assert.assertEquals("Running - Fully Scheduled by DefaultResourceAwareStrategy", cluster.getStatusMap().get(topology3.getId()));
    Map<SupervisorDetails, Double> superToCpu = TestUtilsForResourceAwareScheduler.getSupervisorToCpuUsage(cluster, topologies);
    Map<SupervisorDetails, Double> superToMem = TestUtilsForResourceAwareScheduler.getSupervisorToMemoryUsage(cluster, topologies);
    final Double EPSILON = 0.0001;
    for (SupervisorDetails supervisor : supMap.values()) {
        Double cpuAvailable = supervisor.getTotalCPU();
        Double memAvailable = supervisor.getTotalMemory();
        Double cpuUsed = superToCpu.get(supervisor);
        Double memUsed = superToMem.get(supervisor);
        Assert.assertTrue((Math.abs(memAvailable - memUsed) < EPSILON) || (Math.abs(cpuAvailable - cpuUsed) < EPSILON));
    }
    // end of Test1
    // Test2: Launch topo 1, 2 and 4, they together request a little more mem than available, so one of the 3 topos will not be scheduled
    cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config1);
    topoMap = new HashMap<>();
    topoMap.put(topology1.getId(), topology1);
    topoMap.put(topology2.getId(), topology2);
    topoMap.put(topology4.getId(), topology4);
    topologies = new Topologies(topoMap);
    rs.prepare(config1);
    rs.schedule(topologies, cluster);
    int numTopologiesAssigned = 0;
    if (cluster.getStatusMap().get(topology1.getId()).equals("Running - Fully Scheduled by DefaultResourceAwareStrategy")) {
        numTopologiesAssigned++;
    }
    if (cluster.getStatusMap().get(topology2.getId()).equals("Running - Fully Scheduled by DefaultResourceAwareStrategy")) {
        numTopologiesAssigned++;
    }
    if (cluster.getStatusMap().get(topology4.getId()).equals("Running - Fully Scheduled by DefaultResourceAwareStrategy")) {
        numTopologiesAssigned++;
    }
    Assert.assertEquals(2, numTopologiesAssigned);
    //end of Test2
    //Test3: "Launch topo5 only, both mem and cpu should be exactly used up"
    cluster = new Cluster(iNimbus, supMap, new HashMap<String, SchedulerAssignmentImpl>(), config1);
    topoMap = new HashMap<>();
    topoMap.put(topology5.getId(), topology5);
    topologies = new Topologies(topoMap);
    rs.prepare(config1);
    rs.schedule(topologies, cluster);
    superToCpu = TestUtilsForResourceAwareScheduler.getSupervisorToCpuUsage(cluster, topologies);
    superToMem = TestUtilsForResourceAwareScheduler.getSupervisorToMemoryUsage(cluster, topologies);
    for (SupervisorDetails supervisor : supMap.values()) {
        Double cpuAvailable = supervisor.getTotalCPU();
        Double memAvailable = supervisor.getTotalMemory();
        Double cpuUsed = superToCpu.get(supervisor);
        Double memUsed = superToMem.get(supervisor);
        Assert.assertEquals(cpuAvailable, cpuUsed, 0.0001);
        Assert.assertEquals(memAvailable, memUsed, 0.0001);
    }
//end of Test3
}