Examples with VersionedData org.apache.storm.cluster.VersionedData used on opensource projects

Example 1 with VersionedData

use of org.apache.storm.cluster.VersionedData in project storm by apache.

the class ReadClusterState method run.

@Override
public synchronized void run() {
    try {
        Runnable syncCallback = new EventManagerPushCallback(this, syncSupEventManager);
        List<String> stormIds = stormClusterState.assignments(syncCallback);
        Map<String, VersionedData<Assignment>> assignmentsSnapshot = getAssignmentsSnapshot(stormClusterState, stormIds, assignmentVersions.get(), syncCallback);
        Map<Integer, LocalAssignment> allAssignments = readAssignments(assignmentsSnapshot);
        if (allAssignments == null) {
            //Something odd happened try again later
            return;
        }
        Map<String, List<ProfileRequest>> topoIdToProfilerActions = getProfileActions(stormClusterState, stormIds);
        HashSet<Integer> assignedPorts = new HashSet<>();
        LOG.debug("Synchronizing supervisor");
        LOG.debug("All assignment: {}", allAssignments);
        LOG.debug("Topology Ids -> Profiler Actions {}", topoIdToProfilerActions);
        for (Integer port : allAssignments.keySet()) {
            if (iSuper.confirmAssigned(port)) {
                assignedPorts.add(port);
            }
        }
        HashSet<Integer> allPorts = new HashSet<>(assignedPorts);
        allPorts.addAll(slots.keySet());
        Map<Integer, Set<TopoProfileAction>> filtered = new HashMap<>();
        for (Entry<String, List<ProfileRequest>> entry : topoIdToProfilerActions.entrySet()) {
            String topoId = entry.getKey();
            if (entry.getValue() != null) {
                for (ProfileRequest req : entry.getValue()) {
                    NodeInfo ni = req.get_nodeInfo();
                    if (host.equals(ni.get_node())) {
                        Long port = ni.get_port().iterator().next();
                        Set<TopoProfileAction> actions = filtered.get(port.intValue());
                        if (actions == null) {
                            actions = new HashSet<>();
                            filtered.put(port.intValue(), actions);
                        }
                        actions.add(new TopoProfileAction(topoId, req));
                    }
                }
            }
        }
        for (Integer port : allPorts) {
            Slot slot = slots.get(port);
            if (slot == null) {
                slot = mkSlot(port);
                slots.put(port, slot);
                slot.start();
            }
            slot.setNewAssignment(allAssignments.get(port));
            slot.addProfilerActions(filtered.get(port));
        }
    } catch (Exception e) {
        LOG.error("Failed to Sync Supervisor", e);
        throw new RuntimeException(e);
    }
}

Example 2 with VersionedData

use of org.apache.storm.cluster.VersionedData in project storm by apache.

the class ReadClusterState method readAssignments.

protected Map<Integer, LocalAssignment> readAssignments(Map<String, VersionedData<Assignment>> assignmentsSnapshot) {
    try {
        Map<Integer, LocalAssignment> portLA = new HashMap<>();
        for (Map.Entry<String, VersionedData<Assignment>> assignEntry : assignmentsSnapshot.entrySet()) {
            String topoId = assignEntry.getKey();
            Assignment assignment = assignEntry.getValue().getData();
            Map<Integer, LocalAssignment> portTasks = readMyExecutors(topoId, assignmentId, assignment);
            for (Map.Entry<Integer, LocalAssignment> entry : portTasks.entrySet()) {
                Integer port = entry.getKey();
                LocalAssignment la = entry.getValue();
                if (!portLA.containsKey(port)) {
                    portLA.put(port, la);
                } else {
                    throw new RuntimeException("Should not have multiple topologies assigned to one port " + port + " " + la + " " + portLA);
                }
            }
        }
        readRetry.set(0);
        return portLA;
    } catch (RuntimeException e) {
        if (readRetry.get() > 2) {
            throw e;
        } else {
            readRetry.addAndGet(1);
        }
        LOG.warn("{} : retrying {} of 3", e.getMessage(), readRetry.get());
        return null;
    }
}

Example 3 with VersionedData

use of org.apache.storm.cluster.VersionedData in project storm by apache.

the class WorkerState method refreshConnections.

public void refreshConnections(Runnable callback) throws Exception {
    Integer version = stormClusterState.assignmentVersion(topologyId, callback);
    version = (null == version) ? 0 : version;
    VersionedData<Assignment> assignmentVersion = assignmentVersions.get().get(topologyId);
    Assignment assignment;
    if (null != assignmentVersion && (assignmentVersion.getVersion() == version)) {
        assignment = assignmentVersion.getData();
    } else {
        VersionedData<Assignment> newAssignmentVersion = new VersionedData<>(version, stormClusterState.assignmentInfoWithVersion(topologyId, callback).getData());
        assignmentVersions.getAndUpdate(prev -> {
            Map<String, VersionedData<Assignment>> next = new HashMap<>(prev);
            next.put(topologyId, newAssignmentVersion);
            return next;
        });
        assignment = newAssignmentVersion.getData();
    }
    Set<NodeInfo> neededConnections = new HashSet<>();
    Map<Integer, NodeInfo> newTaskToNodePort = new HashMap<>();
    if (null != assignment) {
        Map<Integer, NodeInfo> taskToNodePort = StormCommon.taskToNodeport(assignment.get_executor_node_port());
        for (Map.Entry<Integer, NodeInfo> taskToNodePortEntry : taskToNodePort.entrySet()) {
            Integer task = taskToNodePortEntry.getKey();
            if (outboundTasks.contains(task)) {
                newTaskToNodePort.put(task, taskToNodePortEntry.getValue());
                if (!taskIds.contains(task)) {
                    neededConnections.add(taskToNodePortEntry.getValue());
                }
            }
        }
    }
    Set<NodeInfo> currentConnections = cachedNodeToPortSocket.get().keySet();
    Set<NodeInfo> newConnections = Sets.difference(neededConnections, currentConnections);
    Set<NodeInfo> removeConnections = Sets.difference(currentConnections, neededConnections);
    // Add new connections atomically
    cachedNodeToPortSocket.getAndUpdate(prev -> {
        Map<NodeInfo, IConnection> next = new HashMap<>(prev);
        for (NodeInfo nodeInfo : newConnections) {
            next.put(nodeInfo, mqContext.connect(topologyId, assignment.get_node_host().get(nodeInfo.get_node()), nodeInfo.get_port().iterator().next().intValue()));
        }
        return next;
    });
    try {
        endpointSocketLock.writeLock().lock();
        cachedTaskToNodePort.set(newTaskToNodePort);
    } finally {
        endpointSocketLock.writeLock().unlock();
    }
    for (NodeInfo nodeInfo : removeConnections) {
        cachedNodeToPortSocket.get().get(nodeInfo).close();
    }
    // Remove old connections atomically
    cachedNodeToPortSocket.getAndUpdate(prev -> {
        Map<NodeInfo, IConnection> next = new HashMap<>(prev);
        removeConnections.forEach(next::remove);
        return next;
    });
}

Example 4 with VersionedData

use of org.apache.storm.cluster.VersionedData in project storm by apache.

the class ReadClusterState method getAssignmentsSnapshot.

protected Map<String, VersionedData<Assignment>> getAssignmentsSnapshot(IStormClusterState stormClusterState, List<String> topoIds, Map<String, VersionedData<Assignment>> localAssignmentVersion, Runnable callback) throws Exception {
    Map<String, VersionedData<Assignment>> updateAssignmentVersion = new HashMap<>();
    for (String topoId : topoIds) {
        Integer recordedVersion = -1;
        Integer version = stormClusterState.assignmentVersion(topoId, callback);
        VersionedData<Assignment> locAssignment = localAssignmentVersion.get(topoId);
        if (locAssignment != null) {
            recordedVersion = locAssignment.getVersion();
        }
        if (version == null) {
        // ignore
        } else if (version == recordedVersion) {
            updateAssignmentVersion.put(topoId, locAssignment);
        } else {
            VersionedData<Assignment> assignmentVersion = stormClusterState.assignmentInfoWithVersion(topoId, callback);
            updateAssignmentVersion.put(topoId, assignmentVersion);
        }
    }
    return updateAssignmentVersion;
}