Examples with Assignment org.apache.storm.generated.Assignment used on opensource projects

Example 16 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class ReadClusterState method getAssignmentsSnapshot.

protected Map<String, VersionedData<Assignment>> getAssignmentsSnapshot(IStormClusterState stormClusterState, List<String> topoIds, Map<String, VersionedData<Assignment>> localAssignmentVersion, Runnable callback) throws Exception {
    Map<String, VersionedData<Assignment>> updateAssignmentVersion = new HashMap<>();
    for (String topoId : topoIds) {
        Integer recordedVersion = -1;
        Integer version = stormClusterState.assignmentVersion(topoId, callback);
        VersionedData<Assignment> locAssignment = localAssignmentVersion.get(topoId);
        if (locAssignment != null) {
            recordedVersion = locAssignment.getVersion();
        }
        if (version == null) {
        // ignore
        } else if (version == recordedVersion) {
            updateAssignmentVersion.put(topoId, locAssignment);
        } else {
            VersionedData<Assignment> assignmentVersion = stormClusterState.assignmentInfoWithVersion(topoId, callback);
            updateAssignmentVersion.put(topoId, assignmentVersion);
        }
    }
    return updateAssignmentVersion;
}

Example 17 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method computeNewTopoToExecToNodePort.

private static Map<String, Map<List<Long>, List<Object>>> computeNewTopoToExecToNodePort(Map<String, SchedulerAssignment> schedAssignments, Map<String, Assignment> existingAssignments) {
    Map<String, Map<List<Long>, List<Object>>> ret = computeTopoToExecToNodePort(schedAssignments);
    // Print some useful information
    if (existingAssignments != null && !existingAssignments.isEmpty()) {
        for (Entry<String, Map<List<Long>, List<Object>>> entry : ret.entrySet()) {
            String topoId = entry.getKey();
            Map<List<Long>, List<Object>> execToNodePort = entry.getValue();
            Assignment assignment = existingAssignments.get(topoId);
            if (assignment == null) {
                continue;
            }
            Map<List<Long>, NodeInfo> old = assignment.get_executor_node_port();
            Map<List<Long>, List<Object>> reassigned = new HashMap<>();
            for (Entry<List<Long>, List<Object>> execAndNodePort : execToNodePort.entrySet()) {
                NodeInfo oldAssigned = old.get(execAndNodePort.getKey());
                String node = (String) execAndNodePort.getValue().get(0);
                Long port = (Long) execAndNodePort.getValue().get(1);
                if (oldAssigned == null || !oldAssigned.get_node().equals(node) || !port.equals(oldAssigned.get_port_iterator().next())) {
                    reassigned.put(execAndNodePort.getKey(), execAndNodePort.getValue());
                }
            }
            if (!reassigned.isEmpty()) {
                int count = (new HashSet<>(execToNodePort.values())).size();
                Set<List<Long>> reExecs = reassigned.keySet();
                LOG.info("Reassigning {} to {} slots", topoId, count);
                LOG.info("Reassign executors: {}", reExecs);
            }
        }
    }
    return ret;
}

Example 18 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method computeTopologyToAliveExecutors.

/**
     * compute a topology-id -> alive executors map
     * @param existingAssignment the current assignments
     * @param topologies the current topologies
     * @param topologyToExecutors the executors for the current topologies
     * @param scratchTopologyId the topology being rebalanced and should be excluded
     * @return the map of topology id to alive executors
     */
private Map<String, Set<List<Integer>>> computeTopologyToAliveExecutors(Map<String, Assignment> existingAssignment, Topologies topologies, Map<String, Set<List<Integer>>> topologyToExecutors, String scratchTopologyId) {
    Map<String, Set<List<Integer>>> ret = new HashMap<>();
    for (Entry<String, Assignment> entry : existingAssignment.entrySet()) {
        String topoId = entry.getKey();
        Assignment assignment = entry.getValue();
        TopologyDetails td = topologies.getById(topoId);
        Set<List<Integer>> allExecutors = topologyToExecutors.get(topoId);
        Set<List<Integer>> aliveExecutors;
        if (topoId.equals(scratchTopologyId)) {
            aliveExecutors = allExecutors;
        } else {
            aliveExecutors = new HashSet<>(aliveExecutors(td, allExecutors, assignment));
        }
        ret.put(topoId, aliveExecutors);
    }
    return ret;
}

Example 19 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class StormClusterStateImpl method assignmentInfoWithVersion.

@Override
public VersionedData<Assignment> assignmentInfoWithVersion(String stormId, Runnable callback) {
    if (callback != null) {
        assignmentInfoWithVersionCallback.put(stormId, callback);
    }
    Assignment assignment = null;
    Integer version = 0;
    VersionedData<byte[]> dataWithVersion = stateStorage.get_data_with_version(ClusterUtils.assignmentPath(stormId), callback != null);
    if (dataWithVersion != null) {
        assignment = ClusterUtils.maybeDeserialize(dataWithVersion.getData(), Assignment.class);
        version = dataWithVersion.getVersion();
    }
    return new VersionedData<Assignment>(version, assignment);
}

Example 20 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class SynchronizeAssignments method assignedAssignmentsToLocal.

private static void assignedAssignmentsToLocal(IStormClusterState clusterState, List<SupervisorAssignments> supervisorAssignments) {
    if (null == supervisorAssignments || supervisorAssignments.isEmpty()) {
        // unknown error, just skip
        return;
    }
    Map<String, byte[]> serAssignments = new HashMap<>();
    for (SupervisorAssignments supervisorAssignment : supervisorAssignments) {
        if (supervisorAssignment == null) {
            // unknown error, just skip
            continue;
        }
        for (Map.Entry<String, Assignment> entry : supervisorAssignment.get_storm_assignment().entrySet()) {
            serAssignments.put(entry.getKey(), Utils.serialize(entry.getValue()));
        }
    }
    clusterState.syncRemoteAssignments(serAssignments);
}