Examples with IStormClusterState org.apache.storm.cluster.IStormClusterState used on opensource projects

Example 6 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method mkAssignments.

private void mkAssignments(String scratchTopoId) throws Exception {
    if (!isLeader()) {
        LOG.info("not a leader, skipping assignments");
        return;
    }
    // get existing assignment (just the topologyToExecutorToNodePort map) -> default to {}
    // filter out ones which have a executor timeout
    // figure out available slots on cluster. add to that the used valid slots to get total slots. figure out how many executors should be in each slot (e.g., 4, 4, 4, 5)
    // only keep existing slots that satisfy one of those slots. for rest, reassign them across remaining slots
    // edge case for slots with no executor timeout but with supervisor timeout... just treat these as valid slots that can be reassigned to. worst comes to worse the executor will timeout and won't assign here next time around
    IStormClusterState state = stormClusterState;
    //read all the topologies
    Map<String, StormBase> bases;
    Map<String, TopologyDetails> tds = new HashMap<>();
    synchronized (submitLock) {
        bases = state.topologyBases();
        for (Iterator<Entry<String, StormBase>> it = bases.entrySet().iterator(); it.hasNext(); ) {
            Entry<String, StormBase> entry = it.next();
            String id = entry.getKey();
            try {
                tds.put(id, readTopologyDetails(id, entry.getValue()));
            } catch (KeyNotFoundException e) {
                //A race happened and it is probably not running
                it.remove();
            }
        }
    }
    Topologies topologies = new Topologies(tds);
    List<String> assignedTopologyIds = state.assignments(null);
    Map<String, Assignment> existingAssignments = new HashMap<>();
    for (String id : assignedTopologyIds) {
        // will be treated as free slot in the scheduler code.
        if (!id.equals(scratchTopoId)) {
            existingAssignments.put(id, state.assignmentInfo(id, null));
        }
    }
    // make the new assignments for topologies
    Map<String, SchedulerAssignment> newSchedulerAssignments = null;
    synchronized (schedLock) {
        newSchedulerAssignments = computeNewSchedulerAssignments(existingAssignments, topologies, bases, scratchTopoId);
        Map<String, Map<List<Long>, List<Object>>> topologyToExecutorToNodePort = computeNewTopoToExecToNodePort(newSchedulerAssignments, existingAssignments);
        for (String id : assignedTopologyIds) {
            if (!topologyToExecutorToNodePort.containsKey(id)) {
                topologyToExecutorToNodePort.put(id, null);
            }
        }
        Map<String, Map<List<Object>, List<Double>>> newAssignedWorkerToResources = computeTopoToNodePortToResources(newSchedulerAssignments);
        int nowSecs = Time.currentTimeSecs();
        Map<String, SupervisorDetails> basicSupervisorDetailsMap = basicSupervisorDetailsMap(state);
        //construct the final Assignments by adding start-times etc into it
        Map<String, Assignment> newAssignments = new HashMap<>();
        for (Entry<String, Map<List<Long>, List<Object>>> entry : topologyToExecutorToNodePort.entrySet()) {
            String topoId = entry.getKey();
            Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            Set<String> allNodes = new HashSet<>();
            if (execToNodePort != null) {
                for (List<Object> nodePort : execToNodePort.values()) {
                    allNodes.add((String) nodePort.get(0));
                }
            }
            Map<String, String> allNodeHost = new HashMap<>();
            if (existingAssignment != null) {
                allNodeHost.putAll(existingAssignment.get_node_host());
            }
            for (String node : allNodes) {
                String host = inimbus.getHostName(basicSupervisorDetailsMap, node);
                if (host != null) {
                    allNodeHost.put(node, host);
                }
            }
            Map<List<Long>, NodeInfo> execNodeInfo = null;
            if (existingAssignment != null) {
                execNodeInfo = existingAssignment.get_executor_node_port();
            }
            List<List<Long>> reassignExecutors = changedExecutors(execNodeInfo, execToNodePort);
            Map<List<Long>, Long> startTimes = new HashMap<>();
            if (existingAssignment != null) {
                startTimes.putAll(existingAssignment.get_executor_start_time_secs());
            }
            for (List<Long> id : reassignExecutors) {
                startTimes.put(id, (long) nowSecs);
            }
            Map<List<Object>, List<Double>> workerToResources = newAssignedWorkerToResources.get(topoId);
            Assignment newAssignment = new Assignment((String) conf.get(Config.STORM_LOCAL_DIR));
            Map<String, String> justAssignedKeys = new HashMap<>(allNodeHost);
            //Modifies justAssignedKeys
            justAssignedKeys.keySet().retainAll(allNodes);
            newAssignment.set_node_host(justAssignedKeys);
            //convert NodePort to NodeInfo (again!!!).
            Map<List<Long>, NodeInfo> execToNodeInfo = new HashMap<>();
            for (Entry<List<Long>, List<Object>> execAndNodePort : execToNodePort.entrySet()) {
                List<Object> nodePort = execAndNodePort.getValue();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                execToNodeInfo.put(execAndNodePort.getKey(), ni);
            }
            newAssignment.set_executor_node_port(execToNodeInfo);
            newAssignment.set_executor_start_time_secs(startTimes);
            //do another conversion (lets just make this all common)
            Map<NodeInfo, WorkerResources> workerResources = new HashMap<>();
            for (Entry<List<Object>, List<Double>> wr : workerToResources.entrySet()) {
                List<Object> nodePort = wr.getKey();
                NodeInfo ni = new NodeInfo();
                ni.set_node((String) nodePort.get(0));
                ni.add_to_port((Long) nodePort.get(1));
                List<Double> r = wr.getValue();
                WorkerResources resources = new WorkerResources();
                resources.set_mem_on_heap(r.get(0));
                resources.set_mem_off_heap(r.get(1));
                resources.set_cpu(r.get(2));
                workerResources.put(ni, resources);
            }
            newAssignment.set_worker_resources(workerResources);
            newAssignments.put(topoId, newAssignment);
        }
        if (!newAssignments.equals(existingAssignments)) {
            LOG.debug("RESETTING id->resources and id->worker-resources cache!");
            idToResources.set(new HashMap<>());
            idToWorkerResources.set(new HashMap<>());
        }
        // only log/set when there's been a change to the assignment
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            //NOT Used TopologyDetails topologyDetails = topologies.getById(topoId);
            if (assignment.equals(existingAssignment)) {
                LOG.debug("Assignment for {} hasn't changed", topoId);
            } else {
                LOG.info("Setting new assignment for topology id {}: {}", topoId, assignment);
                state.setAssignment(topoId, assignment);
            }
        }
        Map<String, Collection<WorkerSlot>> addedSlots = new HashMap<>();
        for (Entry<String, Assignment> entry : newAssignments.entrySet()) {
            String topoId = entry.getKey();
            Assignment assignment = entry.getValue();
            Assignment existingAssignment = existingAssignments.get(topoId);
            if (existingAssignment == null) {
                existingAssignment = new Assignment();
                existingAssignment.set_executor_node_port(new HashMap<>());
                existingAssignment.set_executor_start_time_secs(new HashMap<>());
            }
            Set<WorkerSlot> newSlots = newlyAddedSlots(existingAssignment, assignment);
            addedSlots.put(topoId, newSlots);
        }
        inimbus.assignSlots(topologies, addedSlots);
    }
}

Example 7 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method getClusterInfoImpl.

private ClusterSummary getClusterInfoImpl() throws Exception {
    IStormClusterState state = stormClusterState;
    Map<String, SupervisorInfo> infos = state.allSupervisorInfo();
    List<SupervisorSummary> summaries = new ArrayList<>(infos.size());
    for (Entry<String, SupervisorInfo> entry : infos.entrySet()) {
        summaries.add(makeSupervisorSummary(entry.getKey(), entry.getValue()));
    }
    int uptime = this.uptime.upTime();
    Map<String, StormBase> bases = state.topologyBases();
    List<NimbusSummary> nimbuses = state.nimbuses();
    //update the isLeader field for each nimbus summary
    NimbusInfo leader = leaderElector.getLeader();
    for (NimbusSummary nimbusSummary : nimbuses) {
        nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs()));
        nimbusSummary.set_isLeader(leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port());
    }
    List<TopologySummary> topologySummaries = new ArrayList<>();
    for (Entry<String, StormBase> entry : bases.entrySet()) {
        StormBase base = entry.getValue();
        if (base == null) {
            continue;
        }
        String topoId = entry.getKey();
        Assignment assignment = state.assignmentInfo(topoId, null);
        int numTasks = 0;
        int numExecutors = 0;
        int numWorkers = 0;
        if (assignment != null && assignment.is_set_executor_node_port()) {
            for (List<Long> ids : assignment.get_executor_node_port().keySet()) {
                numTasks += StormCommon.executorIdToTasks(ids).size();
            }
            numExecutors = assignment.get_executor_node_port_size();
            numWorkers = new HashSet<>(assignment.get_executor_node_port().values()).size();
        }
        TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, numExecutors, numWorkers, Time.deltaSecs(base.get_launch_time_secs()), extractStatusStr(base));
        if (base.is_set_owner()) {
            summary.set_owner(base.get_owner());
        }
        String status = idToSchedStatus.get().get(topoId);
        if (status != null) {
            summary.set_sched_status(status);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            summary.set_requested_memonheap(resources.getRequestedMemOnHeap());
            summary.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            summary.set_requested_cpu(resources.getRequestedCpu());
            summary.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            summary.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            summary.set_assigned_cpu(resources.getAssignedCpu());
        }
        summary.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        topologySummaries.add(summary);
    }
    ClusterSummary ret = new ClusterSummary(summaries, topologySummaries, nimbuses);
    ret.set_nimbus_uptime_secs(uptime);
    return ret;
}

Example 8 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method rmTopologyKeys.

@VisibleForTesting
public void rmTopologyKeys(String topoId) {
    BlobStore store = blobStore;
    IStormClusterState state = stormClusterState;
    rmBlobKey(store, ConfigUtils.masterStormJarKey(topoId), state);
    rmBlobKey(store, ConfigUtils.masterStormConfKey(topoId), state);
    rmBlobKey(store, ConfigUtils.masterStormCodeKey(topoId), state);
}

Example 9 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method setupBlobstore.

/**
     * Sets up blobstore state for all current keys.
     * @throws KeyNotFoundException 
     * @throws AuthorizationException 
     */
private void setupBlobstore() throws AuthorizationException, KeyNotFoundException {
    IStormClusterState state = stormClusterState;
    BlobStore store = blobStore;
    Set<String> localKeys = new HashSet<>();
    for (Iterator<String> it = store.listKeys(); it.hasNext(); ) {
        localKeys.add(it.next());
    }
    Set<String> activeKeys = new HashSet<>(state.activeKeys());
    Set<String> activeLocalKeys = new HashSet<>(localKeys);
    activeLocalKeys.retainAll(activeKeys);
    Set<String> keysToDelete = new HashSet<>(localKeys);
    keysToDelete.removeAll(activeKeys);
    NimbusInfo nimbusInfo = nimbusHostPortInfo;
    LOG.debug("Deleting keys not on the zookeeper {}", keysToDelete);
    for (String toDelete : keysToDelete) {
        store.deleteBlob(toDelete, NIMBUS_SUBJECT);
    }
    LOG.debug("Creating list of key entries for blobstore inside zookeeper {} local {}", activeKeys, activeLocalKeys);
    for (String key : activeLocalKeys) {
        try {
            state.setupBlobstore(key, nimbusInfo, getVersionForKey(key, nimbusInfo, conf));
        } catch (KeyNotFoundException e) {
            // invalid key, remove it from blobstore
            store.deleteBlob(key, NIMBUS_SUBJECT);
        }
    }
}

Example 10 with IStormClusterState

use of org.apache.storm.cluster.IStormClusterState in project storm by apache.

the class Nimbus method setupStormCode.

private void setupStormCode(Map<String, Object> conf, String topoId, String tmpJarLocation, Map<String, Object> topoConf, StormTopology topology) throws Exception {
    Subject subject = getSubject();
    IStormClusterState clusterState = stormClusterState;
    BlobStore store = blobStore;
    String jarKey = ConfigUtils.masterStormJarKey(topoId);
    String codeKey = ConfigUtils.masterStormCodeKey(topoId);
    String confKey = ConfigUtils.masterStormConfKey(topoId);
    NimbusInfo hostPortInfo = nimbusHostPortInfo;
    if (tmpJarLocation != null) {
        //in local mode there is no jar
        try (FileInputStream fin = new FileInputStream(tmpJarLocation)) {
            store.createBlob(jarKey, fin, new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
        }
        if (store instanceof LocalFsBlobStore) {
            clusterState.setupBlobstore(jarKey, hostPortInfo, getVersionForKey(jarKey, hostPortInfo, conf));
        }
    }
    store.createBlob(confKey, Utils.toCompressedJsonConf(topoConf), new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
    if (store instanceof LocalFsBlobStore) {
        clusterState.setupBlobstore(confKey, hostPortInfo, getVersionForKey(confKey, hostPortInfo, conf));
    }
    store.createBlob(codeKey, Utils.serialize(topology), new SettableBlobMeta(BlobStoreAclHandler.DEFAULT), subject);
    if (store instanceof LocalFsBlobStore) {
        clusterState.setupBlobstore(codeKey, hostPortInfo, getVersionForKey(codeKey, hostPortInfo, conf));
    }
}