Search in sources :

Example 61 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class SchedulingSearcherState method freeWorkerSlotWithBoundAckers.

/**
 * Free a given workerSlot and all the assigned bound ackers already there.
 *
 * @param node          RasNode which to be freed.
 * @param workerSlot    WorkerSlot on which to schedule.
 */
public void freeWorkerSlotWithBoundAckers(RasNode node, WorkerSlot workerSlot) {
    List<ExecutorDetails> ackers = workerSlotToBoundAckers.get(workerSlot);
    String ackerCompId = Acker.ACKER_COMPONENT_ID;
    if (ackers != null && !ackers.isEmpty()) {
        for (int i = ackers.size() - 1; i >= 0; i--) {
            ExecutorDetails acker = ackers.get(i);
            boundAckers.remove(acker);
            unassignedAckers.addFirst(acker);
            Map<String, Integer> compToAssignmentCount = workerCompAssignmentCnts.get(workerSlot);
            compToAssignmentCount.put(ackerCompId, // decrement worker assignment count
            compToAssignmentCount.getOrDefault(ackerCompId, 0) - 1);
            if (compToAssignmentCount.get(ackerCompId) == 0) {
                compToAssignmentCount.remove(ackerCompId);
            }
            Map<String, Integer> nodeToAssignmentCount = nodeCompAssignmentCnts.get(node);
            nodeToAssignmentCount.put(ackerCompId, // decrement node assignment count
            nodeToAssignmentCount.getOrDefault(ackerCompId, 0) - 1);
            if (nodeToAssignmentCount.get(ackerCompId) == 0) {
                nodeToAssignmentCount.remove(ackerCompId);
            }
        }
        workerSlotToBoundAckers.remove(workerSlot);
        node.free(workerSlot);
    }
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails)

Example 62 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class SchedulingSearcherState method assignCurrentExecutor.

/**
 * Attempt to assign current executor (execIndex points to) to worker and node.
 * Assignment validity check is done before calling this method.
 *
 * @param execToComp Mapping from executor to component name.
 * @param node RasNode on which to schedule.
 * @param workerSlot WorkerSlot on which to schedule.
 */
public void assignCurrentExecutor(Map<ExecutorDetails, String> execToComp, RasNode node, WorkerSlot workerSlot) {
    ExecutorDetails exec = currentExec();
    String comp = execToComp.get(exec);
    LOG.trace("Topology {} Trying assignment of {} {} to {}", topoName, exec, comp, workerSlot);
    // It is possible that this component is already scheduled on this node or worker.  If so when we backtrack we cannot remove it
    Map<String, Integer> compToAssignmentCount = workerCompAssignmentCnts.computeIfAbsent(workerSlot, (k) -> new HashMap<>());
    // increment worker assignment count
    compToAssignmentCount.put(comp, compToAssignmentCount.getOrDefault(comp, 0) + 1);
    okToRemoveFromWorker[execIndex] = true;
    Map<String, Integer> nodeToAssignmentCount = nodeCompAssignmentCnts.computeIfAbsent(node, (k) -> new HashMap<>());
    // increment node assignment count
    nodeToAssignmentCount.put(comp, nodeToAssignmentCount.getOrDefault(comp, 0) + 1);
    okToRemoveFromNode[execIndex] = true;
    node.assignSingleExecutor(workerSlot, exec, td);
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails)

Example 63 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestGenericResourceAwareStrategy method testGenericResourceAwareStrategyWithoutSettingAckerExecutors.

/**
 * Test if the scheduling logic for the GenericResourceAwareStrategy is correct
 * without setting {@link Config#TOPOLOGY_ACKER_EXECUTORS}.
 *
 * Test details refer to {@link TestDefaultResourceAwareStrategy#testDefaultResourceAwareStrategyWithoutSettingAckerExecutors(int)}
 */
@ParameterizedTest
@ValueSource(ints = { -1, 0, 1, 2 })
public void testGenericResourceAwareStrategyWithoutSettingAckerExecutors(int numOfAckersPerWorker) throws InvalidTopologyException {
    int spoutParallelism = 1;
    int boltParallelism = 2;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new TestSpout(), spoutParallelism);
    builder.setBolt("bolt-1", new TestBolt(), boltParallelism).shuffleGrouping("spout");
    builder.setBolt("bolt-2", new TestBolt(), boltParallelism).shuffleGrouping("bolt-1").addResource("gpu.count", 1.0);
    builder.setBolt("bolt-3", new TestBolt(), boltParallelism).shuffleGrouping("bolt-2").addResource("gpu.count", 2.0);
    String topoName = "testTopology";
    StormTopology stormToplogy = builder.createTopology();
    INimbus iNimbus = new INimbusTest();
    Config conf = createGrasClusterConfig(50, 500, 0, null, Collections.emptyMap());
    Map<String, Double> genericResourcesMap = new HashMap<>();
    genericResourcesMap.put("gpu.count", 2.0);
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 200, 2000, genericResourcesMap);
    conf.put(Config.TOPOLOGY_PRIORITY, 0);
    conf.put(Config.TOPOLOGY_NAME, topoName);
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, 2000);
    conf.put(Config.TOPOLOGY_SUBMITTER_USER, "user");
    // Parameterized test on different numOfAckersPerWorker
    if (numOfAckersPerWorker == -1) {
    // Both Config.TOPOLOGY_ACKER_EXECUTORS and Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER are not set
    // Default will be 2 (estimate num of workers) and 1 respectively
    } else {
        conf.put(Config.TOPOLOGY_RAS_ACKER_EXECUTORS_PER_WORKER, numOfAckersPerWorker);
    }
    int estimatedNumWorker = ServerUtils.getEstimatedWorkerCountForRasTopo(conf, stormToplogy);
    Nimbus.setUpAckerExecutorConfigs(topoName, conf, conf, estimatedNumWorker);
    conf.put(Config.TOPOLOGY_ACKER_RESOURCES_ONHEAP_MEMORY_MB, 250);
    conf.put(Config.TOPOLOGY_ACKER_CPU_PCORE_PERCENT, 50);
    TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, genExecsAndComps(StormCommon.systemTopology(conf, stormToplogy)), currentTime, "user");
    Topologies topologies = new Topologies(topo);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf);
    scheduler = new ResourceAwareScheduler();
    scheduler.prepare(conf, new StormMetricsRegistry());
    scheduler.schedule(topologies, cluster);
    // We need to have 3 slots on 3 separate hosts. The topology needs 6 GPUs 3500 MB memory and 350% CPU
    // The bolt-3 instances must be on separate nodes because they each need 2 GPUs.
    // The bolt-2 instances must be on the same node as they each need 1 GPU
    // (this assumes that we are packing the components to avoid fragmentation).
    // The bolt-1 and spout instances fill in the rest.
    // Ordered execs: [[6, 6], [2, 2], [4, 4], [5, 5], [1, 1], [3, 3], [0, 0]]
    // Ackers: [[8, 8], [7, 7]] (+ [[9, 9], [10, 10]] when numOfAckersPerWorker=2)
    HashSet<HashSet<ExecutorDetails>> expectedScheduling = new HashSet<>();
    if (numOfAckersPerWorker == -1 || numOfAckersPerWorker == 1) {
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3 - 500 MB, 50% CPU, 2 GPU
        new ExecutorDetails(3, 3))));
        // Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-2 - 500 MB, 50% CPU, 1 GPU
        new ExecutorDetails(6, 6), // bolt-1 - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(2, 2), // bolt-2 - 500 MB, 50% CPU, 1 GPU
        new ExecutorDetails(5, 5), // acker - 250 MB, 50% CPU, 0 GPU
        new ExecutorDetails(8, 8))));
        // Total 1750 MB, 200% CPU, 2 GPU -> this node has 250 MB, 0% CPU, 0 GPU left
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3 500 MB, 50% cpu, 2 GPU
        new ExecutorDetails(4, 4), // bolt-1 - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(1, 1), // Spout - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(0, 0), // acker - 250 MB, 50% CPU, 0 GPU
        new ExecutorDetails(7, 7))));
    // Total 1750 MB, 200% CPU, 2 GPU -> this node has 250 MB, 0% CPU, 0 GPU left
    } else if (numOfAckersPerWorker == 0) {
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3 - 500 MB, 50% CPU, 2 GPU
        new ExecutorDetails(3, 3))));
        // Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-2 - 500 MB, 50% CPU, 1 GPU
        new ExecutorDetails(6, 6), // bolt-1 - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(2, 2), // bolt-2 - 500 MB, 50% CPU, 1 GPU
        new ExecutorDetails(5, 5), // bolt-1 - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(1, 1))));
        // Total 2000 MB, 200% CPU, 2 GPU -> this node has 0 MB, 0% CPU, 0 GPU left
        expectedScheduling.add(new HashSet<>(Arrays.asList(// Spout - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(0, 0), // bolt-3 500 MB, 50% cpu, 2 GPU
        new ExecutorDetails(4, 4))));
    // Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left
    } else if (numOfAckersPerWorker == 2) {
        expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3 - 500 MB, 50% CPU, 2 GPU
        new ExecutorDetails(3, 3))));
        // Total 500 MB, 50% CPU, 2 - GPU -> this node has 1500 MB, 150% cpu, 0 GPU left
        expectedScheduling.add(new HashSet<>(Arrays.asList(// acker - 250 MB, 50% CPU, 0 GPU
        new ExecutorDetails(7, 7), // acker - 250 MB, 50% CPU, 0 GPU
        new ExecutorDetails(8, 8), // bolt-2 - 500 MB, 50% CPU, 1 GPU
        new ExecutorDetails(6, 6), // bolt-1 - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(2, 2))));
        // Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left
        expectedScheduling.add(new HashSet<>(Arrays.asList(// acker- 250 MB, 50% CPU, 0 GPU
        new ExecutorDetails(9, 9), // acker- 250 MB, 50% CPU, 0 GPU
        new ExecutorDetails(10, 10), // bolt-1 - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(1, 1), // bolt-3 500 MB, 50% cpu, 2 GPU
        new ExecutorDetails(4, 4))));
        // Total 1500 MB, 200% CPU, 2 GPU -> this node has 500 MB, 0% CPU, 0 GPU left
        expectedScheduling.add(new HashSet<>(Arrays.asList(// Spout - 500 MB, 50% CPU, 0 GPU
        new ExecutorDetails(0, 0), // bolt-2 - 500 MB, 50% CPU, 1 GPU
        new ExecutorDetails(5, 5))));
    // Total 1000 MB, 100% CPU, 2 GPU -> this node has 1000 MB, 100% CPU, 0 GPU left
    }
    HashSet<HashSet<ExecutorDetails>> foundScheduling = new HashSet<>();
    SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id");
    for (Collection<ExecutorDetails> execs : assignment.getSlotToExecutors().values()) {
        foundScheduling.add(new HashSet<>(execs));
    }
    assertEquals(expectedScheduling, foundScheduling);
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) HashMap(java.util.HashMap) DaemonConfig(org.apache.storm.DaemonConfig) Config(org.apache.storm.Config) StormTopology(org.apache.storm.generated.StormTopology) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) HashSet(java.util.HashSet) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 64 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestGenericResourceAwareStrategy method testGenericResourceAwareStrategyInFavorOfShuffle.

/**
 * test if the scheduling logic for the GenericResourceAwareStrategy (when in favor of shuffle) is correct.
 */
@Test
public void testGenericResourceAwareStrategyInFavorOfShuffle() throws InvalidTopologyException {
    int spoutParallelism = 1;
    int boltParallelism = 2;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new TestSpout(), spoutParallelism);
    builder.setBolt("bolt-1", new TestBolt(), boltParallelism).shuffleGrouping("spout");
    builder.setBolt("bolt-2", new TestBolt(), boltParallelism).shuffleGrouping("bolt-1").addResource("gpu.count", 1.0);
    builder.setBolt("bolt-3", new TestBolt(), boltParallelism).shuffleGrouping("bolt-2").addResource("gpu.count", 2.0);
    StormTopology stormToplogy = builder.createTopology();
    INimbus iNimbus = new INimbusTest();
    Config conf = createGrasClusterConfig(50, 250, 250, null, Collections.emptyMap());
    Map<String, Double> genericResourcesMap = new HashMap<>();
    genericResourcesMap.put("gpu.count", 2.0);
    Map<String, SupervisorDetails> supMap = genSupervisors(4, 4, 200, 2000, genericResourcesMap);
    conf.put(Config.TOPOLOGY_PRIORITY, 0);
    conf.put(Config.TOPOLOGY_NAME, "testTopology");
    conf.put(Config.TOPOLOGY_WORKER_MAX_HEAP_SIZE_MB, Double.MAX_VALUE);
    conf.put(Config.TOPOLOGY_SUBMITTER_USER, "user");
    conf.put(Config.TOPOLOGY_RAS_ORDER_EXECUTORS_BY_PROXIMITY_NEEDS, true);
    TopologyDetails topo = new TopologyDetails("testTopology-id", conf, stormToplogy, 0, genExecsAndComps(StormCommon.systemTopology(conf, stormToplogy)), currentTime, "user");
    Topologies topologies = new Topologies(topo);
    Cluster cluster = new Cluster(iNimbus, new ResourceMetrics(new StormMetricsRegistry()), supMap, new HashMap<>(), topologies, conf);
    ResourceAwareScheduler rs = new ResourceAwareScheduler();
    rs.prepare(conf, new StormMetricsRegistry());
    rs.schedule(topologies, cluster);
    // Sorted execs: [[0, 0], [2, 2], [6, 6], [4, 4], [1, 1], [5, 5], [3, 3], [7, 7]]
    // Ackers: [[7, 7]]]
    HashSet<HashSet<ExecutorDetails>> expectedScheduling = new HashSet<>();
    expectedScheduling.add(new HashSet<>(Arrays.asList(// spout
    new ExecutorDetails(0, 0), // bolt-1
    new ExecutorDetails(2, 2), // bolt-2
    new ExecutorDetails(6, 6), // acker
    new ExecutorDetails(7, 7))));
    expectedScheduling.add(new HashSet<>(Arrays.asList(// bolt-3
    new ExecutorDetails(4, 4), // bolt-1
    new ExecutorDetails(1, 1))));
    // bolt-2
    expectedScheduling.add(new HashSet<>(Arrays.asList(new ExecutorDetails(5, 5))));
    // bolt-3
    expectedScheduling.add(new HashSet<>(Arrays.asList(new ExecutorDetails(3, 3))));
    HashSet<HashSet<ExecutorDetails>> foundScheduling = new HashSet<>();
    SchedulerAssignment assignment = cluster.getAssignmentById("testTopology-id");
    for (Collection<ExecutorDetails> execs : assignment.getSlotToExecutors().values()) {
        foundScheduling.add(new HashSet<>(execs));
    }
    assertEquals(expectedScheduling, foundScheduling);
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) TopologyBuilder(org.apache.storm.topology.TopologyBuilder) HashMap(java.util.HashMap) DaemonConfig(org.apache.storm.DaemonConfig) Config(org.apache.storm.Config) StormTopology(org.apache.storm.generated.StormTopology) StormMetricsRegistry(org.apache.storm.metric.StormMetricsRegistry) ResourceAwareScheduler(org.apache.storm.scheduler.resource.ResourceAwareScheduler) TestUtilsForResourceAwareScheduler(org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler) ResourceMetrics(org.apache.storm.scheduler.resource.normalization.ResourceMetrics) Topologies(org.apache.storm.scheduler.Topologies) SupervisorDetails(org.apache.storm.scheduler.SupervisorDetails) HashSet(java.util.HashSet) Cluster(org.apache.storm.scheduler.Cluster) INimbus(org.apache.storm.scheduler.INimbus) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) SchedulerAssignment(org.apache.storm.scheduler.SchedulerAssignment) Test(org.junit.Test) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Example 65 with ExecutorDetails

use of org.apache.storm.scheduler.ExecutorDetails in project storm by apache.

the class TestConstraintSolverStrategy method testConstraintSolverForceBacktrackWithSpreadCoLocation.

@Test
public void testConstraintSolverForceBacktrackWithSpreadCoLocation() {
    // to find an answer.
    if (CO_LOCATION_CNT > 1 && !consolidatedConfigFlag) {
        LOG.info("INFO: Skipping Test {} with {}={} (required 1), and consolidatedConfigFlag={} (required false)", "testConstraintSolverForceBacktrackWithSpreadCoLocation", ConstraintSolverConfig.CONSTRAINT_TYPE_MAX_NODE_CO_LOCATION_CNT, CO_LOCATION_CNT, consolidatedConfigFlag);
        return;
    }
    ConstraintSolverStrategy cs = new ConstraintSolverStrategy() {

        protected void prepareForScheduling(Cluster cluster, TopologyDetails topologyDetails) {
            super.prepareForScheduling(cluster, topologyDetails);
            // set a reversing execSorter instance
            IExecSorter execSorter = new ExecSorterByConstraintSeverity(cluster, topologyDetails) {

                @Override
                public List<ExecutorDetails> sortExecutors(Set<ExecutorDetails> unassignedExecutors) {
                    List<ExecutorDetails> tmp = super.sortExecutors(unassignedExecutors);
                    List<ExecutorDetails> reversed = new ArrayList<>();
                    while (!tmp.isEmpty()) {
                        reversed.add(0, tmp.remove(0));
                    }
                    return reversed;
                }
            };
            setExecSorter(execSorter);
        }
    };
    basicUnitTestWithKillAndRecover(cs, BACKTRACK_BOLT_PARALLEL, CO_LOCATION_CNT);
}
Also used : ExecutorDetails(org.apache.storm.scheduler.ExecutorDetails) HashSet(java.util.HashSet) Set(java.util.Set) ArrayList(java.util.ArrayList) Cluster(org.apache.storm.scheduler.Cluster) TopologyDetails(org.apache.storm.scheduler.TopologyDetails) ExecSorterByConstraintSeverity(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.ExecSorterByConstraintSeverity) IExecSorter(org.apache.storm.scheduler.resource.strategies.scheduling.sorter.IExecSorter) Test(org.junit.Test)

Aggregations

ExecutorDetails (org.apache.storm.scheduler.ExecutorDetails)72 HashMap (java.util.HashMap)50 TopologyDetails (org.apache.storm.scheduler.TopologyDetails)42 WorkerSlot (org.apache.storm.scheduler.WorkerSlot)41 SchedulerAssignment (org.apache.storm.scheduler.SchedulerAssignment)36 ArrayList (java.util.ArrayList)35 Map (java.util.Map)34 Cluster (org.apache.storm.scheduler.Cluster)31 Config (org.apache.storm.Config)29 HashSet (java.util.HashSet)28 List (java.util.List)28 SupervisorDetails (org.apache.storm.scheduler.SupervisorDetails)28 Topologies (org.apache.storm.scheduler.Topologies)23 LinkedList (java.util.LinkedList)21 INimbus (org.apache.storm.scheduler.INimbus)21 Collection (java.util.Collection)20 StormMetricsRegistry (org.apache.storm.metric.StormMetricsRegistry)19 StormTopology (org.apache.storm.generated.StormTopology)18 TestUtilsForResourceAwareScheduler (org.apache.storm.scheduler.resource.TestUtilsForResourceAwareScheduler)18 ResourceMetrics (org.apache.storm.scheduler.resource.normalization.ResourceMetrics)18