Examples with TestWordSpout org.apache.storm.testing.TestWordSpout used on opensource projects

Example 26 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class TestResourceAwareScheduler method testScheduleResilience.

@Test
public void testScheduleResilience() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout1", new TestWordSpout(), 3);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config1 = new Config();
    config1.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config1, stormTopology1, 3, executorMap1, 0, "user");
    TopologyBuilder builder2 = new TopologyBuilder();
    builder2.setSpout("wordSpout2", new TestWordSpout(), 2);
    StormTopology stormTopology2 = builder2.createTopology();
    Config config2 = new Config();
    config2.putAll(defaultTopologyConf);
    // memory requirement is large enough so that two executors can not be fully assigned to one node
    config2.put(Config.TOPOLOGY_COMPONENT_RESOURCES_ONHEAP_MEMORY_MB, 1280.0);
    Map<ExecutorDetails, String> executorMap2 = genExecsAndComps(stormTopology2);
    TopologyDetails topology2 = new TopologyDetails("topology2", config2, stormTopology2, 2, executorMap2, 0, "user");
    // Test1: When a worker fails, RAS does not alter existing assignments on healthy workers
    scheduler = new ResourceAwareScheduler();
    Topologies topologies = new Topologies(topology2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    scheduler.prepare(config1, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    SchedulerAssignment assignment = cluster.getAssignmentById(topology2.getId());
    // pick a worker to mock as failed
    WorkerSlot failedWorker = new ArrayList<>(assignment.getSlots()).get(0);
    Map<ExecutorDetails, WorkerSlot> executorToSlot = assignment.getExecutorToSlot();
    List<ExecutorDetails> failedExecutors = new ArrayList<>();
    for (Map.Entry<ExecutorDetails, WorkerSlot> entry : executorToSlot.entrySet()) {
        if (entry.getValue().equals(failedWorker)) {
            failedExecutors.add(entry.getKey());
        }
    }
    for (ExecutorDetails executor : failedExecutors) {
        // remove executor details assigned to the failed worker
        executorToSlot.remove(executor);
    }
    Map<ExecutorDetails, WorkerSlot> copyOfOldMapping = new HashMap<>(executorToSlot);
    Set<ExecutorDetails> healthyExecutors = copyOfOldMapping.keySet();
    scheduler.schedule(topologies, cluster);
    SchedulerAssignment newAssignment = cluster.getAssignmentById(topology2.getId());
    Map<ExecutorDetails, WorkerSlot> newExecutorToSlot = newAssignment.getExecutorToSlot();
    for (ExecutorDetails executor : healthyExecutors) {
        assertEquals(copyOfOldMapping.get(executor), newExecutorToSlot.get(executor));
    }
    assertFalse(cluster.needsSchedulingRas(topology2));
    assertTrue(cluster.getStatusMap().get(topology2.getId()).startsWith("Running - Fully Scheduled by DefaultResourceAwareStrategy"));
    // end of Test1
    // Test2: When a supervisor fails, RAS does not alter existing assignments
    executorToSlot = new HashMap<>();
    executorToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("r000s000", 0));
    executorToSlot.put(new ExecutorDetails(1, 1), new WorkerSlot("r000s000", 1));
    executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", 1));
    Map<String, SchedulerAssignment> existingAssignments = new HashMap<>();
    assignment = new SchedulerAssignmentImpl(topology1.getId(), executorToSlot, null, null);
    existingAssignments.put(topology1.getId(), assignment);
    copyOfOldMapping = new HashMap<>(executorToSlot);
    Set<ExecutorDetails> existingExecutors = copyOfOldMapping.keySet();
    Map<String, SupervisorDetails> supMap1 = new HashMap<>(supMap);
    // mock the supervisor r000s000 as a failed supervisor
    supMap1.remove("r000s000");
    topologies = new Topologies(topology1);
    Cluster cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap1, existingAssignments, topologies, config1);
    scheduler.schedule(topologies, cluster1);
    newAssignment = cluster1.getAssignmentById(topology1.getId());
    newExecutorToSlot = newAssignment.getExecutorToSlot();
    for (ExecutorDetails executor : existingExecutors) {
        assertEquals(copyOfOldMapping.get(executor), newExecutorToSlot.get(executor));
    }
    assertEquals("Fully Scheduled", cluster1.getStatusMap().get(topology1.getId()));
    // end of Test2
    // Test3: When a supervisor and a worker on it fails, RAS does not alter existing assignments
    executorToSlot = new HashMap<>();
    // the worker to orphan
    executorToSlot.put(new ExecutorDetails(0, 0), new WorkerSlot("r000s000", 1));
    // the worker that fails
    executorToSlot.put(new ExecutorDetails(1, 1), new WorkerSlot("r000s000", 2));
    // the healthy worker
    executorToSlot.put(new ExecutorDetails(2, 2), new WorkerSlot("r000s001", 1));
    existingAssignments = new HashMap<>();
    assignment = new SchedulerAssignmentImpl(topology1.getId(), executorToSlot, null, null);
    existingAssignments.put(topology1.getId(), assignment);
    // delete one worker of r000s000 (failed) from topo1 assignment to enable actual schedule for testing
    executorToSlot.remove(new ExecutorDetails(1, 1));
    copyOfOldMapping = new HashMap<>(executorToSlot);
    // namely the two eds on the orphaned worker and the healthy worker
    existingExecutors = copyOfOldMapping.keySet();
    supMap1 = new HashMap<>(supMap);
    // mock the supervisor r000s000 as a failed supervisor
    supMap1.remove("r000s000");
    topologies = new Topologies(topology1);
    cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap1, existingAssignments, topologies, config1);
    scheduler.schedule(topologies, cluster1);
    newAssignment = cluster1.getAssignmentById(topology1.getId());
    newExecutorToSlot = newAssignment.getExecutorToSlot();
    for (ExecutorDetails executor : existingExecutors) {
        assertEquals(copyOfOldMapping.get(executor), newExecutorToSlot.get(executor));
    }
    assertFalse(cluster1.needsSchedulingRas(topology1));
    assertEquals("Fully Scheduled", cluster1.getStatusMap().get(topology1.getId()));
    // end of Test3
    // Test4: Scheduling a new topology does not disturb other assignments unnecessarily
    topologies = new Topologies(topology1);
    cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    scheduler.schedule(topologies, cluster1);
    assignment = cluster1.getAssignmentById(topology1.getId());
    executorToSlot = assignment.getExecutorToSlot();
    copyOfOldMapping = new HashMap<>(executorToSlot);
    topologies = addTopologies(topologies, topology2);
    cluster1 = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config1);
    scheduler.schedule(topologies, cluster1);
    newAssignment = cluster1.getAssignmentById(topology1.getId());
    newExecutorToSlot = newAssignment.getExecutorToSlot();
    for (ExecutorDetails executor : copyOfOldMapping.keySet()) {
        assertEquals(copyOfOldMapping.get(executor), newExecutorToSlot.get(executor));
    }
    assertFalse(cluster1.needsSchedulingRas(topology1));
    assertFalse(cluster1.needsSchedulingRas(topology2));
    String expectedStatusPrefix = "Running - Fully Scheduled by DefaultResourceAwareStrategy";
    assertTrue(cluster1.getStatusMap().get(topology1.getId()).startsWith(expectedStatusPrefix));
    assertTrue(cluster1.getStatusMap().get(topology2.getId()).startsWith(expectedStatusPrefix));
}

Example 27 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class TestResourceAwareScheduler method testTopologySetCpuAndMemLoad.

@Test
public void testTopologySetCpuAndMemLoad() {
    INimbus iNimbus = new INimbusTest();
    Map<String, SupervisorDetails> supMap = genSupervisors(2, 2, 400, 2000);
    // a topology with multiple spouts
    TopologyBuilder builder1 = new TopologyBuilder();
    builder1.setSpout("wordSpout", new TestWordSpout(), 1).setCPULoad(20.0).setMemoryLoad(200.0);
    builder1.setBolt("wordCountBolt", new TestWordCounter(), 1).shuffleGrouping("wordSpout").setCPULoad(20.0).setMemoryLoad(200.0);
    StormTopology stormTopology1 = builder1.createTopology();
    Config config = new Config();
    config.putAll(defaultTopologyConf);
    Map<ExecutorDetails, String> executorMap1 = genExecsAndComps(stormTopology1);
    TopologyDetails topology1 = new TopologyDetails("topology1", config, stormTopology1, 0, executorMap1, 0, "user");
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    scheduler = rs;
    Topologies topologies = new Topologies(topology1);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    rs.prepare(config, new StormMetricsRegistry());
    rs.schedule(topologies, cluster);
    SchedulerAssignment assignment1 = cluster.getAssignmentById(topology1.getId());
    Map<WorkerSlot, WorkerResources> assignedSlots1 = assignment1.getScheduledResources();
    double assignedMemory = 0.0;
    double assignedCpu = 0.0;
    Set<String> nodesIDs1 = new HashSet<>();
    for (Entry<WorkerSlot, WorkerResources> entry : assignedSlots1.entrySet()) {
        WorkerResources wr = entry.getValue();
        nodesIDs1.add(entry.getKey().getNodeId());
        assignedMemory += wr.get_mem_on_heap() + wr.get_mem_off_heap();
        assignedCpu += wr.get_cpu();
    }
    Collection<ExecutorDetails> executors1 = assignment1.getExecutors();
    assertEquals(1, assignedSlots1.size());
    assertEquals(1, nodesIDs1.size());
    assertEquals(2, executors1.size());
    assertEquals(400.0, assignedMemory, 0.001);
    assertEquals(40.0, assignedCpu, 0.001);
    assertFalse(cluster.needsSchedulingRas(topology1));
    String expectedStatusPrefix = "Running - Fully Scheduled by DefaultResourceAwareStrategy";
    assertTrue(cluster.getStatusMap().get(topology1.getId()).startsWith(expectedStatusPrefix));
}

Example 28 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class ResourceAwareExampleTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    // A topology can set resources in terms of CPU and Memory for each component
    // These can be chained (like with setting the CPU requirement)
    SpoutDeclarer spout = builder.setSpout("word", new TestWordSpout(), 10).setCPULoad(20);
    // Or done separately like with setting the
    // onheap and offheap memory requirement
    spout.setMemoryLoad(64, 16);
    // On heap memory is used to help calculate the heap of the java process for the worker
    // off heap memory is for things like JNI memory allocated off heap, or when using the
    // ShellBolt or ShellSpout.  In this case the 16 MB of off heap is just as an example
    // as we are not using it.
    // Some times a Bolt or Spout will have some memory that is shared between the instances
    // These are typically caches, but could be anything like a static database that is memory
    // mapped into the processes. These can be declared separately and added to the bolts and
    // spouts that use them.  Or if only one uses it they can be created inline with the add
    SharedOnHeap exclaimCache = new SharedOnHeap(100, "exclaim-cache");
    SharedOffHeapWithinNode notImplementedButJustAnExample = new SharedOffHeapWithinNode(500, "not-implemented-node-level-cache");
    // If CPU or memory is not set the values stored in topology.component.resources.onheap.memory.mb,
    // topology.component.resources.offheap.memory.mb and topology.component.cpu.pcore.percent
    // will be used instead
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word").addSharedMemory(exclaimCache);
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1").setMemoryLoad(100).addSharedMemory(exclaimCache).addSharedMemory(notImplementedButJustAnExample);
    Config conf = new Config();
    conf.setDebug(true);
    // Under RAS the number of workers is determined by the scheduler and the settings in the conf are ignored
    // conf.setNumWorkers(3);
    // Instead the scheduler lets you set the maximum heap size for any worker.
    conf.setTopologyWorkerMaxHeapSize(1024.0);
    // The scheduler generally will try to pack executors into workers until the max heap size is met, but
    // this can vary depending on the specific scheduling strategy selected.
    // The reason for this is to try and balance the maximum pause time GC might take (which is larger for larger heaps)
    // against better performance because of not needing to serialize/deserialize tuples.
    // The priority of a topology describes the importance of the topology in decreasing importance
    // starting from 0 (i.e. 0 is the highest priority and the priority importance decreases as the priority number increases).
    // Recommended range of 0-29 but no hard limit set.
    // If there are not enough resources in a cluster the priority in combination with how far over a guarantees
    // a user is will decide which topologies are run and which ones are not.
    conf.setTopologyPriority(29);
    // set to use the default resource aware strategy when using the MultitenantResourceAwareBridgeScheduler
    conf.setTopologyStrategy("org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy");
    String topoName = "test";
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}

Example 29 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class MultipleLoggerTopology method main.

public static void main(String[] args) throws Exception {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationLoggingBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationLoggingBolt(), 2).shuffleGrouping("exclaim1");
    Config conf = new Config();
    conf.setDebug(true);
    String topoName = MultipleLoggerTopology.class.getName();
    if (args != null && args.length > 0) {
        topoName = args[0];
    }
    conf.setNumWorkers(2);
    StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
}

Example 30 with TestWordSpout

use of org.apache.storm.testing.TestWordSpout in project storm by apache.

the class TopologyIntegrationTest method mkInvalidateTopology2.

private StormTopology mkInvalidateTopology2() {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("1", new TestWordSpout(true), 3);
    builder.setBolt("2", new TestWordCounter(), 4).fieldsGrouping("1", new Fields("non-exists-field"));
    return builder.createTopology();
}