Examples with IEvictionStrategy org.apache.storm.scheduler.resource.strategies.eviction.IEvictionStrategy used on opensource projects

Example 1 with IEvictionStrategy

use of org.apache.storm.scheduler.resource.strategies.eviction.IEvictionStrategy in project storm by apache.

the class ResourceAwareScheduler method scheduleTopology.

public void scheduleTopology(TopologyDetails td) {
    User topologySubmitter = this.schedulingState.userMap.get(td.getTopologySubmitter());
    if (this.schedulingState.cluster.getUnassignedExecutors(td).size() > 0) {
        LOG.debug("/********Scheduling topology {} from User {}************/", td.getName(), topologySubmitter);
        SchedulingState schedulingState = checkpointSchedulingState();
        IStrategy rasStrategy = null;
        try {
            rasStrategy = (IStrategy) Utils.newInstance((String) td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY));
        } catch (RuntimeException e) {
            LOG.error("failed to create instance of IStrategy: {} with error: {}! Topology {} will not be scheduled.", td.getName(), td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY), e.getMessage());
            topologySubmitter = cleanup(schedulingState, td);
            topologySubmitter.moveTopoFromPendingToInvalid(td);
            this.schedulingState.cluster.setStatus(td.getId(), "Unsuccessful in scheduling - failed to create instance of topology strategy " + td.getConf().get(Config.TOPOLOGY_SCHEDULER_STRATEGY) + ". Please check logs for details");
            return;
        }
        IEvictionStrategy evictionStrategy = null;
        while (true) {
            SchedulingResult result = null;
            try {
                // Need to re prepare scheduling strategy with cluster and topologies in case scheduling state was restored
                // Pass in a copy of scheduling state since the scheduling strategy should not be able to be able to make modifications to
                // the state of cluster directly
                rasStrategy.prepare(new SchedulingState(this.schedulingState));
                result = rasStrategy.schedule(td);
            } catch (Exception ex) {
                LOG.error(String.format("Exception thrown when running strategy %s to schedule topology %s. Topology will not be scheduled!", rasStrategy.getClass().getName(), td.getName()), ex);
                topologySubmitter = cleanup(schedulingState, td);
                topologySubmitter.moveTopoFromPendingToInvalid(td);
                this.schedulingState.cluster.setStatus(td.getId(), "Unsuccessful in scheduling - Exception thrown when running strategy {}" + rasStrategy.getClass().getName() + ". Please check logs for details");
            }
            LOG.debug("scheduling result: {}", result);
            if (result != null && result.isValid()) {
                if (result.isSuccess()) {
                    try {
                        if (mkAssignment(td, result.getSchedulingResultMap())) {
                            topologySubmitter.moveTopoFromPendingToRunning(td);
                            this.schedulingState.cluster.setStatus(td.getId(), "Running - " + result.getMessage());
                        } else {
                            topologySubmitter = this.cleanup(schedulingState, td);
                            topologySubmitter.moveTopoFromPendingToAttempted(td);
                            this.schedulingState.cluster.setStatus(td.getId(), "Unsuccessful in scheduling - Unable to assign executors to nodes. Please check logs for details");
                        }
                    } catch (IllegalStateException ex) {
                        LOG.error("Unsuccessful in scheduling - IllegalStateException thrown when attempting to assign executors to nodes.", ex);
                        topologySubmitter = cleanup(schedulingState, td);
                        topologySubmitter.moveTopoFromPendingToAttempted(td);
                        this.schedulingState.cluster.setStatus(td.getId(), "Unsuccessful in scheduling - IllegalStateException thrown when attempting to assign executors to nodes. Please check log for details.");
                    }
                    break;
                } else {
                    if (result.getStatus() == SchedulingStatus.FAIL_NOT_ENOUGH_RESOURCES) {
                        if (evictionStrategy == null) {
                            try {
                                evictionStrategy = (IEvictionStrategy) Utils.newInstance((String) this.conf.get(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY));
                            } catch (RuntimeException e) {
                                LOG.error("failed to create instance of eviction strategy: {} with error: {}! No topology eviction will be done.", this.conf.get(Config.RESOURCE_AWARE_SCHEDULER_EVICTION_STRATEGY), e.getMessage());
                                topologySubmitter.moveTopoFromPendingToAttempted(td);
                                break;
                            }
                        }
                        boolean madeSpace = false;
                        try {
                            //need to re prepare since scheduling state might have been restored
                            evictionStrategy.prepare(this.schedulingState);
                            madeSpace = evictionStrategy.makeSpaceForTopo(td);
                        } catch (Exception ex) {
                            LOG.error(String.format("Exception thrown when running eviction strategy %s to schedule topology %s. No evictions will be done! Error: %s", evictionStrategy.getClass().getName(), td.getName(), ex.getClass().getName()), ex);
                            topologySubmitter = cleanup(schedulingState, td);
                            topologySubmitter.moveTopoFromPendingToAttempted(td);
                            break;
                        }
                        if (!madeSpace) {
                            LOG.debug("Could not make space for topo {} will move to attempted", td);
                            topologySubmitter = cleanup(schedulingState, td);
                            topologySubmitter.moveTopoFromPendingToAttempted(td);
                            this.schedulingState.cluster.setStatus(td.getId(), "Not enough resources to schedule - " + result.getErrorMessage());
                            break;
                        }
                        continue;
                    } else if (result.getStatus() == SchedulingStatus.FAIL_INVALID_TOPOLOGY) {
                        topologySubmitter = cleanup(schedulingState, td);
                        topologySubmitter.moveTopoFromPendingToInvalid(td, this.schedulingState.cluster);
                        break;
                    } else {
                        topologySubmitter = cleanup(schedulingState, td);
                        topologySubmitter.moveTopoFromPendingToAttempted(td, this.schedulingState.cluster);
                        break;
                    }
                }
            } else {
                LOG.warn("Scheduling results returned from topology {} is not vaild! Topology with be ignored.", td.getName());
                topologySubmitter = cleanup(schedulingState, td);
                topologySubmitter.moveTopoFromPendingToInvalid(td, this.schedulingState.cluster);
                break;
            }
        }
    } else {
        LOG.warn("Topology {} is already fully scheduled!", td.getName());
        topologySubmitter.moveTopoFromPendingToRunning(td);
        if (this.schedulingState.cluster.getStatusMap().get(td.getId()) == null || this.schedulingState.cluster.getStatusMap().get(td.getId()).equals("")) {
            this.schedulingState.cluster.setStatus(td.getId(), "Fully Scheduled");
        }
    }
}