Search in sources :

Example 1 with DNSToSwitchMapping

use of org.apache.storm.networktopography.DNSToSwitchMapping in project storm by apache.

the class TestNodeSorterHostProximity method testMultipleRacksWithFavoritism.

/**
 * Test whether strategy will choose correct rack.
 */
@Test
public void testMultipleRacksWithFavoritism() {
    final Map<String, SupervisorDetails> supMap = new HashMap<>();
    final int numRacks = 1;
    final int numSupersPerRack = 10;
    final int numPortsPerSuper = 4;
    final int numZonesPerHost = 2;
    int rackStartNum = 0;
    int supStartNum = 0;
    final Map<String, SupervisorDetails> supMapRack0 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400, 8000, Collections.emptyMap(), 1.0);
    // generate another rack of supervisors with less resources
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack1 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 200, 4000, Collections.emptyMap(), 1.0);
    // generate some supervisors that are depleted of one resource
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack2 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 0, 8000, Collections.emptyMap(), 1.0);
    // generate some that has a lot of memory but little of cpu
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack3 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 10, 8000 * 2 + 4000, Collections.emptyMap(), 1.0);
    // generate some that has a lot of cpu but little of memory
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack4 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400 + 200 + 10, 1000, Collections.emptyMap(), 1.0);
    supMap.putAll(supMapRack0);
    supMap.putAll(supMapRack1);
    supMap.putAll(supMapRack2);
    supMap.putAll(supMapRack3);
    supMap.putAll(supMapRack4);
    Config config = createClusterConfig(100, 500, 500, null);
    config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    INimbus iNimbus = new INimbusTest();
    // create test DNSToSwitchMapping plugin
    TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4);
    Config t1Conf = new Config();
    t1Conf.putAll(config);
    final List<String> t1FavoredHostNames = Arrays.asList("host-41", "host-42", "host-43");
    t1Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, t1FavoredHostNames);
    final List<String> t1UnfavoredHostIds = Arrays.asList("host-1", "host-2", "host-3");
    t1Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, t1UnfavoredHostIds);
    // generate topologies
    TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    Config t2Conf = new Config();
    t2Conf.putAll(config);
    t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", "host-33"));
    t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", "host-13"));
    TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    Topologies topologies = new Topologies(topo1, topo2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    List<String> supHostnames = new LinkedList<>();
    for (SupervisorDetails sup : supMap.values()) {
        supHostnames.add(sup.getHost());
    }
    Map<String, List<String>> rackToHosts = testDNSToSwitchMapping.getRackToHosts();
    cluster.setNetworkTopography(rackToHosts);
    NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS);
    nodeSorter.prepare(null);
    List<ObjectResourcesItem> sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors.toList());
    String rackSummaries = sortedRacks.stream().map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())).collect(Collectors.joining("\n\t"));
    Iterator<ObjectResourcesItem> it = sortedRacks.iterator();
    // Ranked first since rack-000 has the most balanced set of resources
    Assert.assertEquals("rack-000 should be ordered first", "rack-000", it.next().id);
    // Ranked second since rack-1 has a balanced set of resources but less than rack-0
    Assert.assertEquals("rack-001 should be ordered second", "rack-001", it.next().id);
    // Ranked third since rack-4 has a lot of cpu but not a lot of memory
    Assert.assertEquals("rack-004 should be ordered third", "rack-004", it.next().id);
    // Ranked fourth since rack-3 has alot of memory but not cpu
    Assert.assertEquals("rack-003 should be ordered fourth", "rack-003", it.next().id);
    // Ranked last since rack-2 has not cpu resources
    Assert.assertEquals("rack-00s2 should be ordered fifth", "rack-002", it.next().id);
}
Also used : Arrays(java.util.Arrays) NormalizedResourceRequest(org.apache.storm.scheduler.resource.normalization.NormalizedResourceRequest) DefaultResourceAwareStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy) LoggerFactory(org.slf4j.LoggerFactory) RasNodes(org.apache.storm.scheduler.resource.RasNodes) INimbus(org.apache.storm.scheduler.INimbus) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) Map(java.util.Map) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) DNSToSwitchMapping(org.apache.storm.networktopography.DNSToSwitchMapping) Collection(java.util.Collection) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) Set(java.util.Set) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) List(java.util.List) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) Assert.assertFalse(org.junit.Assert.assertFalse) Config(org.apache.storm.Config) ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) IScheduler(org.apache.storm.scheduler.IScheduler) HashMap(java.util.HashMap) BaseResourceAwareStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.BaseResourceAwareStrategy) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Topologies(org.apache.storm.scheduler.Topologies) NormalizedResourcesExtension(org.apache.storm.scheduler.resource.normalization.NormalizedResourcesExtension) StreamSupport(java.util.stream.StreamSupport) LinkedList(java.util.LinkedList) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) GenericResourceAwareStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) Cluster(org.apache.storm.scheduler.Cluster) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) AtomicLong(java.util.concurrent.atomic.AtomicLong) TreeMap(java.util.TreeMap) ObjectResourcesItem(org.apache.storm.scheduler.resource.strategies.scheduling.ObjectResourcesItem) Assert(org.junit.Assert) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) HashMap(java.util.HashMap) Config(org.apache.storm.Config) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) ObjectResourcesItem(org.apache.storm.scheduler.resource.strategies.scheduling.ObjectResourcesItem) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) Topologies(org.apache.storm.scheduler.Topologies) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) LinkedList(java.util.LinkedList) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Test(org.junit.jupiter.api.Test)

Example 2 with DNSToSwitchMapping

use of org.apache.storm.networktopography.DNSToSwitchMapping in project storm by apache.

the class TestNodeSorterHostProximity method testMultipleRacksOrderedByCapacity.

/**
 * Racks should be returned in order of decreasing capacity.
 */
@Test
public void testMultipleRacksOrderedByCapacity() {
    final Map<String, SupervisorDetails> supMap = new HashMap<>();
    final int numRacks = 1;
    final int numSupersPerRack = 10;
    final int numPortsPerSuper = 4;
    final int numZonesPerHost = 1;
    final double numaResourceMultiplier = 1.0;
    int rackStartNum = 0;
    int supStartNum = 0;
    final Map<String, SupervisorDetails> supMapRack0 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 600, 8000 - rackStartNum, Collections.emptyMap(), numaResourceMultiplier);
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack1 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 500, 8000 - rackStartNum, Collections.emptyMap(), numaResourceMultiplier);
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack2 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 400, 8000 - rackStartNum, Collections.emptyMap(), numaResourceMultiplier);
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack3 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 300, 8000 - rackStartNum, Collections.emptyMap(), numaResourceMultiplier);
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack4 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 200, 8000 - rackStartNum, Collections.emptyMap(), numaResourceMultiplier);
    // too small to hold topology
    supStartNum += numSupersPerRack;
    final Map<String, SupervisorDetails> supMapRack5 = genSupervisorsWithRacksAndNuma(numRacks, numSupersPerRack, numZonesPerHost, numPortsPerSuper, rackStartNum++, supStartNum, 100, 8000 - rackStartNum, Collections.singletonMap("gpu.count", 0.0), numaResourceMultiplier);
    supMap.putAll(supMapRack0);
    supMap.putAll(supMapRack1);
    supMap.putAll(supMapRack2);
    supMap.putAll(supMapRack3);
    supMap.putAll(supMapRack4);
    supMap.putAll(supMapRack5);
    Config config = createClusterConfig(100, 500, 500, null);
    config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    INimbus iNimbus = new INimbusTest();
    // create test DNSToSwitchMapping plugin
    TestDNSToSwitchMapping testDNSToSwitchMapping = new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4, supMapRack5);
    // generate topologies
    TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    Topologies topologies = new Topologies(topo1, topo2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    cluster.setNetworkTopography(testDNSToSwitchMapping.getRackToHosts());
    NodeSorterHostProximity nodeSorter = new NodeSorterHostProximity(cluster, topo1);
    nodeSorter.prepare(null);
    List<ObjectResourcesItem> sortedRacks = StreamSupport.stream(nodeSorter.getSortedRacks().spliterator(), false).collect(Collectors.toList());
    String rackSummaries = sortedRacks.stream().map(x -> String.format("Rack %s -> scheduled-cnt %d, min-avail %f, avg-avail %f, cpu %f, mem %f", x.id, nodeSorter.getScheduledExecCntByRackId().getOrDefault(x.id, new AtomicInteger(-1)).get(), x.minResourcePercent, x.avgResourcePercent, x.availableResources.getTotalCpu(), x.availableResources.getTotalMemoryMb())).collect(Collectors.joining("\n\t"));
    NormalizedResourceRequest topoResourceRequest = topo1.getApproximateTotalResources();
    String topoRequest = String.format("Topo %s, approx-requested-resources %s", topo1.getId(), topoResourceRequest.toString());
    Iterator<ObjectResourcesItem> it = sortedRacks.iterator();
    Assert.assertEquals(topoRequest + "\n\t" + rackSummaries + "\nRack-000 should be ordered first since it has the largest capacity", "rack-000", it.next().id);
    Assert.assertEquals(topoRequest + "\n\t" + rackSummaries + "\nrack-001 should be ordered second since it smaller than rack-000", "rack-001", it.next().id);
    Assert.assertEquals(topoRequest + "\n\t" + rackSummaries + "\nrack-002 should be ordered third since it is smaller than rack-001", "rack-002", it.next().id);
    Assert.assertEquals(topoRequest + "\n\t" + rackSummaries + "\nrack-003 should be ordered fourth since it since it is smaller than rack-002", "rack-003", it.next().id);
    Assert.assertEquals(topoRequest + "\n\t" + rackSummaries + "\nrack-004 should be ordered fifth since it since it is smaller than rack-003", "rack-004", it.next().id);
    Assert.assertEquals(topoRequest + "\n\t" + rackSummaries + "\nrack-005 should be ordered last since it since it is has smallest capacity", "rack-005", it.next().id);
}
Also used : Arrays(java.util.Arrays) NormalizedResourceRequest(org.apache.storm.scheduler.resource.normalization.NormalizedResourceRequest) DefaultResourceAwareStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.DefaultResourceAwareStrategy) LoggerFactory(org.slf4j.LoggerFactory) RasNodes(org.apache.storm.scheduler.resource.RasNodes) INimbus(org.apache.storm.scheduler.INimbus) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) Map(java.util.Map) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) DNSToSwitchMapping(org.apache.storm.networktopography.DNSToSwitchMapping) Collection(java.util.Collection) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) Set(java.util.Set) Collectors(java.util.stream.Collectors) Test(org.junit.jupiter.api.Test) List(java.util.List) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) Assert.assertFalse(org.junit.Assert.assertFalse) Config(org.apache.storm.Config) ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) IScheduler(org.apache.storm.scheduler.IScheduler) HashMap(java.util.HashMap) BaseResourceAwareStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.BaseResourceAwareStrategy) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Topologies(org.apache.storm.scheduler.Topologies) NormalizedResourcesExtension(org.apache.storm.scheduler.resource.normalization.NormalizedResourcesExtension) StreamSupport(java.util.stream.StreamSupport) LinkedList(java.util.LinkedList) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) GenericResourceAwareStrategy(org.apache.storm.scheduler.resource.strategies.scheduling.GenericResourceAwareStrategy) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Assert.assertNotEquals(org.junit.Assert.assertNotEquals) Cluster(org.apache.storm.scheduler.Cluster) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) AtomicLong(java.util.concurrent.atomic.AtomicLong) TreeMap(java.util.TreeMap) ObjectResourcesItem(org.apache.storm.scheduler.resource.strategies.scheduling.ObjectResourcesItem) Assert(org.junit.Assert) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) NormalizedResourceRequest(org.apache.storm.scheduler.resource.normalization.NormalizedResourceRequest) HashMap(java.util.HashMap) Config(org.apache.storm.Config) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) ObjectResourcesItem(org.apache.storm.scheduler.resource.strategies.scheduling.ObjectResourcesItem) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Test(org.junit.jupiter.api.Test)

Example 3 with DNSToSwitchMapping

use of org.apache.storm.networktopography.DNSToSwitchMapping in project storm by apache.

the class Cluster method getNetworkTopography.

/*
    * Note: Make sure the proper conf was passed into the Cluster constructor before calling this function
    * It tries to load the proper network topography detection plugin specified in the config.
    */
public Map<String, List<String>> getNetworkTopography() {
    if (networkTopography == null) {
        networkTopography = new HashMap<String, List<String>>();
        ArrayList<String> supervisorHostNames = new ArrayList<String>();
        for (SupervisorDetails s : supervisors.values()) {
            supervisorHostNames.add(s.getHost());
        }
        String clazz = (String) conf.get(Config.STORM_NETWORK_TOPOGRAPHY_PLUGIN);
        DNSToSwitchMapping topographyMapper = (DNSToSwitchMapping) Utils.newInstance(clazz);
        Map<String, String> resolvedSuperVisors = topographyMapper.resolve(supervisorHostNames);
        for (Map.Entry<String, String> entry : resolvedSuperVisors.entrySet()) {
            String hostName = entry.getKey();
            String rack = entry.getValue();
            List<String> nodesForRack = networkTopography.get(rack);
            if (nodesForRack == null) {
                nodesForRack = new ArrayList<String>();
                networkTopography.put(rack, nodesForRack);
            }
            nodesForRack.add(hostName);
        }
    }
    return networkTopography;
}
Also used : ArrayList(java.util.ArrayList) DNSToSwitchMapping(org.apache.storm.networktopography.DNSToSwitchMapping) List(java.util.List) ArrayList(java.util.ArrayList) Map(java.util.Map) HashMap(java.util.HashMap)

Example 4 with DNSToSwitchMapping

use of org.apache.storm.networktopography.DNSToSwitchMapping in project storm by apache.

the class TestDefaultResourceAwareStrategy method testMultipleRacks.

/**
 * Test whether strategy will choose correct rack
 */
@Test
public void testMultipleRacks() {
    final Map<String, SupervisorDetails> supMap = new HashMap<>();
    final Map<String, SupervisorDetails> supMapRack0 = genSupervisors(10, 4, 0, 400, 8000);
    // generate another rack of supervisors with less resources
    final Map<String, SupervisorDetails> supMapRack1 = genSupervisors(10, 4, 10, 200, 4000);
    // generate some supervisors that are depleted of one resource
    final Map<String, SupervisorDetails> supMapRack2 = genSupervisors(10, 4, 20, 0, 8000);
    // generate some that has alot of memory but little of cpu
    final Map<String, SupervisorDetails> supMapRack3 = genSupervisors(10, 4, 30, 10, 8000 * 2 + 4000);
    // generate some that has alot of cpu but little of memory
    final Map<String, SupervisorDetails> supMapRack4 = genSupervisors(10, 4, 40, 400 + 200 + 10, 1000);
    // Generate some that have neither resource, to verify that the strategy will prioritize this last
    // Also put a generic resource with 0 value in the resources list, to verify that it doesn't affect the sorting
    final Map<String, SupervisorDetails> supMapRack5 = genSupervisors(10, 4, 50, 0.0, 0.0, Collections.singletonMap("gpu.count", 0.0));
    supMap.putAll(supMapRack0);
    supMap.putAll(supMapRack1);
    supMap.putAll(supMapRack2);
    supMap.putAll(supMapRack3);
    supMap.putAll(supMapRack4);
    supMap.putAll(supMapRack5);
    Config config = createClusterConfig(100, 500, 500, null);
    config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    INimbus iNimbus = new INimbusTest();
    // create test DNSToSwitchMapping plugin
    DNSToSwitchMapping TestNetworkTopographyPlugin = new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4, supMapRack5);
    // generate topologies
    TopologyDetails topo1 = genTopology("topo-1", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    TopologyDetails topo2 = genTopology("topo-2", config, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    Topologies topologies = new Topologies(topo1, topo2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    List<String> supHostnames = new LinkedList<>();
    for (SupervisorDetails sup : supMap.values()) {
        supHostnames.add(sup.getHost());
    }
    Map<String, List<String>> rackToNodes = new HashMap<>();
    Map<String, String> resolvedSuperVisors = TestNetworkTopographyPlugin.resolve(supHostnames);
    for (Map.Entry<String, String> entry : resolvedSuperVisors.entrySet()) {
        String hostName = entry.getKey();
        String rack = entry.getValue();
        rackToNodes.computeIfAbsent(rack, rid -> new ArrayList<>()).add(hostName);
    }
    cluster.setNetworkTopography(rackToNodes);
    DefaultResourceAwareStrategyOld rs = new DefaultResourceAwareStrategyOld();
    rs.prepareForScheduling(cluster, topo1);
    INodeSorter nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS);
    nodeSorter.prepare(null);
    Iterable<ObjectResourcesItem> sortedRacks = nodeSorter.getSortedRacks();
    Iterator<ObjectResourcesItem> it = sortedRacks.iterator();
    // Ranked first since rack-0 has the most balanced set of resources
    Assert.assertEquals("rack-0 should be ordered first", "rack-0", it.next().id);
    // Ranked second since rack-1 has a balanced set of resources but less than rack-0
    Assert.assertEquals("rack-1 should be ordered second", "rack-1", it.next().id);
    // Ranked third since rack-4 has a lot of cpu but not a lot of memory
    Assert.assertEquals("rack-4 should be ordered third", "rack-4", it.next().id);
    // Ranked fourth since rack-3 has alot of memory but not cpu
    Assert.assertEquals("rack-3 should be ordered fourth", "rack-3", it.next().id);
    // Ranked fifth since rack-2 has not cpu resources
    Assert.assertEquals("rack-2 should be ordered fifth", "rack-2", it.next().id);
    // Ranked last since rack-5 has neither CPU nor memory available
    assertEquals("Rack-5 should be ordered sixth", "rack-5", it.next().id);
    SchedulingResult schedulingResult = rs.schedule(cluster, topo1);
    assert (schedulingResult.isSuccess());
    SchedulerAssignment assignment = cluster.getAssignmentById(topo1.getId());
    for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) {
        // make sure all workers on scheduled in rack-0
        Assert.assertEquals("assert worker scheduled on rack-0", "rack-0", resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()));
    }
    Assert.assertEquals("All executors in topo-1 scheduled", 0, cluster.getUnassignedExecutors(topo1).size());
    // Test if topology is already partially scheduled on one rack
    Iterator<ExecutorDetails> executorIterator = topo2.getExecutors().iterator();
    List<String> nodeHostnames = rackToNodes.get("rack-1");
    for (int i = 0; i < topo2.getExecutors().size() / 2; i++) {
        String nodeHostname = nodeHostnames.get(i % nodeHostnames.size());
        RasNode node = rs.hostnameToNodes(nodeHostname).get(0);
        WorkerSlot targetSlot = node.getFreeSlots().iterator().next();
        ExecutorDetails targetExec = executorIterator.next();
        // to keep track of free slots
        node.assign(targetSlot, topo2, Arrays.asList(targetExec));
    }
    rs = new DefaultResourceAwareStrategyOld();
    // schedule topo2
    schedulingResult = rs.schedule(cluster, topo2);
    assert (schedulingResult.isSuccess());
    assignment = cluster.getAssignmentById(topo2.getId());
    for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) {
        // make sure all workers on scheduled in rack-1
        Assert.assertEquals("assert worker scheduled on rack-1", "rack-1", resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()));
    }
    Assert.assertEquals("All executors in topo-2 scheduled", 0, cluster.getUnassignedExecutors(topo2).size());
}
Also used : Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) INimbus(org.apache.storm.scheduler.INimbus) SupervisorResources(org.apache.storm.scheduler.SupervisorResources) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) Matchers.closeTo(org.hamcrest.Matchers.closeTo) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) Map(java.util.Map) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) NodeSorterHostProximity(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorterHostProximity) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) DNSToSwitchMapping(org.apache.storm.networktopography.DNSToSwitchMapping) Collection(java.util.Collection) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) Collectors(java.util.stream.Collectors) SharedOnHeap(org.apache.storm.topology.SharedOnHeap) Test(org.junit.jupiter.api.Test) WorkerResources(org.apache.storm.generated.WorkerResources) List(java.util.List) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) Entry(java.util.Map.Entry) Config(org.apache.storm.Config) Matchers.is(org.hamcrest.Matchers.is) InvalidTopologyException(org.apache.storm.generated.InvalidTopologyException) StormCommon(org.apache.storm.daemon.StormCommon) ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) IScheduler(org.apache.storm.scheduler.IScheduler) RasNode(org.apache.storm.scheduler.resource.RasNode) SharedOffHeapWithinNode(org.apache.storm.topology.SharedOffHeapWithinNode) NodeSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorter) EnumSource(org.junit.jupiter.params.provider.EnumSource) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Topologies(org.apache.storm.scheduler.Topologies) ServerUtils(org.apache.storm.utils.ServerUtils) StormTopology(org.apache.storm.generated.StormTopology) NormalizedResourcesExtension(org.apache.storm.scheduler.resource.normalization.NormalizedResourcesExtension) LinkedList(java.util.LinkedList) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) ValueSource(org.junit.jupiter.params.provider.ValueSource) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) SharedOffHeapWithinWorker(org.apache.storm.topology.SharedOffHeapWithinWorker) INodeSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.INodeSorter) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) TopologyResources(org.apache.storm.daemon.nimbus.TopologyResources) Cluster(org.apache.storm.scheduler.Cluster) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) SchedulingResult(org.apache.storm.scheduler.resource.SchedulingResult) Nimbus(org.apache.storm.daemon.nimbus.Nimbus) AfterEach(org.junit.jupiter.api.AfterEach) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Assert(org.junit.Assert) Collections(java.util.Collections) ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) Config(org.apache.storm.Config) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) ArrayList(java.util.ArrayList) NodeSorterHostProximity(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorterHostProximity) SchedulingResult(org.apache.storm.scheduler.resource.SchedulingResult) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) INodeSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.INodeSorter) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) RasNode(org.apache.storm.scheduler.resource.RasNode) DNSToSwitchMapping(org.apache.storm.networktopography.DNSToSwitchMapping) Topologies(org.apache.storm.scheduler.Topologies) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) LinkedList(java.util.LinkedList) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 5 with DNSToSwitchMapping

use of org.apache.storm.networktopography.DNSToSwitchMapping in project storm by apache.

the class TestDefaultResourceAwareStrategy method testMultipleRacksWithFavoritism.

/**
 * Test whether strategy will choose correct rack
 */
@Test
public void testMultipleRacksWithFavoritism() {
    final Map<String, SupervisorDetails> supMap = new HashMap<>();
    final Map<String, SupervisorDetails> supMapRack0 = genSupervisors(10, 4, 0, 400, 8000);
    // generate another rack of supervisors with less resources
    final Map<String, SupervisorDetails> supMapRack1 = genSupervisors(10, 4, 10, 200, 4000);
    // generate some supervisors that are depleted of one resource
    final Map<String, SupervisorDetails> supMapRack2 = genSupervisors(10, 4, 20, 0, 8000);
    // generate some that has alot of memory but little of cpu
    final Map<String, SupervisorDetails> supMapRack3 = genSupervisors(10, 4, 30, 10, 8000 * 2 + 4000);
    // generate some that has alot of cpu but little of memory
    final Map<String, SupervisorDetails> supMapRack4 = genSupervisors(10, 4, 40, 400 + 200 + 10, 1000);
    supMap.putAll(supMapRack0);
    supMap.putAll(supMapRack1);
    supMap.putAll(supMapRack2);
    supMap.putAll(supMapRack3);
    supMap.putAll(supMapRack4);
    Config config = createClusterConfig(100, 500, 500, null);
    config.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    INimbus iNimbus = new INimbusTest();
    // create test DNSToSwitchMapping plugin
    DNSToSwitchMapping TestNetworkTopographyPlugin = new TestDNSToSwitchMapping(supMapRack0, supMapRack1, supMapRack2, supMapRack3, supMapRack4);
    Config t1Conf = new Config();
    t1Conf.putAll(config);
    final List<String> t1FavoredHostNames = Arrays.asList("host-41", "host-42", "host-43");
    t1Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, t1FavoredHostNames);
    final List<String> t1UnfavoredHostIds = Arrays.asList("host-1", "host-2", "host-3");
    t1Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, t1UnfavoredHostIds);
    // generate topologies
    TopologyDetails topo1 = genTopology("topo-1", t1Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    Config t2Conf = new Config();
    t2Conf.putAll(config);
    t2Conf.put(Config.TOPOLOGY_SCHEDULER_FAVORED_NODES, Arrays.asList("host-31", "host-32", "host-33"));
    t2Conf.put(Config.TOPOLOGY_SCHEDULER_UNFAVORED_NODES, Arrays.asList("host-11", "host-12", "host-13"));
    TopologyDetails topo2 = genTopology("topo-2", t2Conf, 8, 0, 2, 0, CURRENT_TIME - 2, 10, "user");
    Topologies topologies = new Topologies(topo1, topo2);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, config);
    List<String> supHostnames = new LinkedList<>();
    for (SupervisorDetails sup : supMap.values()) {
        supHostnames.add(sup.getHost());
    }
    Map<String, List<String>> rackToNodes = new HashMap<>();
    Map<String, String> resolvedSuperVisors = TestNetworkTopographyPlugin.resolve(supHostnames);
    for (Map.Entry<String, String> entry : resolvedSuperVisors.entrySet()) {
        String hostName = entry.getKey();
        String rack = entry.getValue();
        List<String> nodesForRack = rackToNodes.get(rack);
        if (nodesForRack == null) {
            nodesForRack = new ArrayList<>();
            rackToNodes.put(rack, nodesForRack);
        }
        nodesForRack.add(hostName);
    }
    cluster.setNetworkTopography(rackToNodes);
    DefaultResourceAwareStrategyOld rs = new DefaultResourceAwareStrategyOld();
    rs.prepareForScheduling(cluster, topo1);
    INodeSorter nodeSorter = new NodeSorterHostProximity(cluster, topo1, BaseResourceAwareStrategy.NodeSortType.DEFAULT_RAS);
    nodeSorter.prepare(null);
    Iterable<ObjectResourcesItem> sortedRacks = nodeSorter.getSortedRacks();
    Iterator<ObjectResourcesItem> it = sortedRacks.iterator();
    // Ranked first since rack-0 has the most balanced set of resources
    Assert.assertEquals("rack-0 should be ordered first", "rack-0", it.next().id);
    // Ranked second since rack-1 has a balanced set of resources but less than rack-0
    Assert.assertEquals("rack-1 should be ordered second", "rack-1", it.next().id);
    // Ranked third since rack-4 has a lot of cpu but not a lot of memory
    Assert.assertEquals("rack-4 should be ordered third", "rack-4", it.next().id);
    // Ranked fourth since rack-3 has alot of memory but not cpu
    Assert.assertEquals("rack-3 should be ordered fourth", "rack-3", it.next().id);
    // Ranked last since rack-2 has not cpu resources
    Assert.assertEquals("rack-2 should be ordered fifth", "rack-2", it.next().id);
    SchedulingResult schedulingResult = rs.schedule(cluster, topo1);
    assert (schedulingResult.isSuccess());
    SchedulerAssignment assignment = cluster.getAssignmentById(topo1.getId());
    for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) {
        String hostName = rs.idToNode(ws.getNodeId()).getHostname();
        String rackId = resolvedSuperVisors.get(hostName);
        Assert.assertTrue(ws + " is neither on a favored node " + t1FavoredHostNames + " nor the highest priority rack (rack-0)", t1FavoredHostNames.contains(hostName) || "rack-0".equals(rackId));
        Assert.assertFalse(ws + " is a part of an unfavored node " + t1UnfavoredHostIds, t1UnfavoredHostIds.contains(hostName));
    }
    Assert.assertEquals("All executors in topo-1 scheduled", 0, cluster.getUnassignedExecutors(topo1).size());
    // Test if topology is already partially scheduled on one rack
    Iterator<ExecutorDetails> executorIterator = topo2.getExecutors().iterator();
    List<String> nodeHostnames = rackToNodes.get("rack-1");
    for (int i = 0; i < topo2.getExecutors().size() / 2; i++) {
        String nodeHostname = nodeHostnames.get(i % nodeHostnames.size());
        RasNode node = rs.hostnameToNodes(nodeHostname).get(0);
        WorkerSlot targetSlot = node.getFreeSlots().iterator().next();
        ExecutorDetails targetExec = executorIterator.next();
        // to keep track of free slots
        node.assign(targetSlot, topo2, Arrays.asList(targetExec));
    }
    rs = new DefaultResourceAwareStrategyOld();
    // schedule topo2
    schedulingResult = rs.schedule(cluster, topo2);
    assert (schedulingResult.isSuccess());
    assignment = cluster.getAssignmentById(topo2.getId());
    for (WorkerSlot ws : assignment.getSlotToExecutors().keySet()) {
        // make sure all workers on scheduled in rack-1
        // The favored nodes would have put it on a different rack, but because that rack does not have free space to run the
        // topology it falls back to this rack
        Assert.assertEquals("assert worker scheduled on rack-1", "rack-1", resolvedSuperVisors.get(rs.idToNode(ws.getNodeId()).getHostname()));
    }
    Assert.assertEquals("All executors in topo-2 scheduled", 0, cluster.getUnassignedExecutors(topo2).size());
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashMap(java.util.HashMap) Config(org.apache.storm.Config) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) NodeSorterHostProximity(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.NodeSorterHostProximity) SchedulingResult(org.apache.storm.scheduler.resource.SchedulingResult) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) INodeSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.INodeSorter) WorkerSlot(org.apache.storm.scheduler.WorkerSlot) RasNode(org.apache.storm.scheduler.resource.RasNode) DNSToSwitchMapping(org.apache.storm.networktopography.DNSToSwitchMapping) Topologies(org.apache.storm.scheduler.Topologies) List(java.util.List) ArrayList(java.util.ArrayList) LinkedList(java.util.LinkedList) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) LinkedList(java.util.LinkedList) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) Map(java.util.Map) HashMap(java.util.HashMap) Test(org.junit.jupiter.api.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

ArrayList (java.util.ArrayList)7 HashMap (java.util.HashMap)7 List (java.util.List)7 Map (java.util.Map)7 DNSToSwitchMapping (org.apache.storm.networktopography.DNSToSwitchMapping)7 LinkedList (java.util.LinkedList)6 Config (org.apache.storm.Config)6 StormMetricsRegistry (org.apache.storm.metric.StormMetricsRegistry)6 Cluster (org.apache.storm.scheduler.Cluster)6 ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)6 INimbus (org.apache.storm.scheduler.INimbus)6 SchedulerAssignment (org.apache.storm.scheduler.SchedulerAssignment)6 SupervisorDetails (org.apache.storm.scheduler.SupervisorDetails)6 Topologies (org.apache.storm.scheduler.Topologies)6 TopologyDetails (org.apache.storm.scheduler.TopologyDetails)6 WorkerSlot (org.apache.storm.scheduler.WorkerSlot)6 ResourceMetrics (org.apache.storm.scheduler.resource.normalization.ResourceMetrics)6 Arrays (java.util.Arrays)5 Collection (java.util.Collection)5 Collections (java.util.Collections)5