Examples with Assignment org.apache.storm.generated.Assignment used on opensource projects

Example 11 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Supervisor method createSupervisorIface.

private org.apache.storm.generated.Supervisor.Iface createSupervisorIface() {
    return new org.apache.storm.generated.Supervisor.Iface() {

        @Override
        public void sendSupervisorAssignments(SupervisorAssignments assignments) throws AuthorizationException, TException {
            checkAuthorization("sendSupervisorAssignments");
            LOG.info("Got an assignments from master, will start to sync with assignments: {}", assignments);
            SynchronizeAssignments syn = new SynchronizeAssignments(getSupervisor(), assignments, getReadClusterState());
            getEventManger().add(syn);
        }

        @Override
        public Assignment getLocalAssignmentForStorm(String id) throws NotAliveException, AuthorizationException, TException {
            Map<String, Object> topoConf = null;
            try {
                topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
            } catch (IOException e) {
                LOG.warn("Topology config is not localized yet...");
            }
            checkAuthorization(id, topoConf, "getLocalAssignmentForStorm");
            Assignment assignment = getStormClusterState().assignmentInfo(id, null);
            if (null == assignment) {
                throw new WrappedNotAliveException("No local assignment assigned for storm: " + id + " for node: " + getHostName());
            }
            return assignment;
        }

        @Override
        public void sendSupervisorWorkerHeartbeat(SupervisorWorkerHeartbeat heartbeat) throws AuthorizationException, NotAliveException, TException {
            // do nothing except validate heartbeat for now.
            String id = heartbeat.get_storm_id();
            Map<String, Object> topoConf = null;
            try {
                topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
            } catch (IOException e) {
                LOG.warn("Topology config is not localized yet...");
                throw new WrappedNotAliveException(id + " does not appear to be alive, you should probably exit");
            }
            checkAuthorization(id, topoConf, "sendSupervisorWorkerHeartbeat");
        }
    };
}

Example 12 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class Nimbus method getSupervisorPageInfo.

@Override
public SupervisorPageInfo getSupervisorPageInfo(String superId, String host, boolean includeSys) throws NotAliveException, AuthorizationException, TException {
    try {
        getSupervisorPageInfoCalls.mark();
        IStormClusterState state = stormClusterState;
        Map<String, SupervisorInfo> superInfos = state.allSupervisorInfo();
        Map<String, List<String>> hostToSuperId = new HashMap<>();
        for (Entry<String, SupervisorInfo> entry : superInfos.entrySet()) {
            String h = entry.getValue().get_hostname();
            List<String> superIds = hostToSuperId.get(h);
            if (superIds == null) {
                superIds = new ArrayList<>();
                hostToSuperId.put(h, superIds);
            }
            superIds.add(entry.getKey());
        }
        List<String> supervisorIds = null;
        if (superId == null) {
            supervisorIds = hostToSuperId.get(host);
        } else {
            supervisorIds = Arrays.asList(superId);
        }
        SupervisorPageInfo pageInfo = new SupervisorPageInfo();
        Map<String, Assignment> topoToAssignment = state.assignmentsInfo();
        for (String sid : supervisorIds) {
            SupervisorInfo info = superInfos.get(sid);
            LOG.info("SIDL {} SI: {} ALL: {}", sid, info, superInfos);
            SupervisorSummary supSum = makeSupervisorSummary(sid, info);
            pageInfo.add_to_supervisor_summaries(supSum);
            List<String> superTopologies = topologiesOnSupervisor(topoToAssignment, sid);
            Set<String> userTopologies = filterAuthorized("getTopology", superTopologies);
            for (String topoId : superTopologies) {
                CommonTopoInfo common = getCommonTopoInfo(topoId, "getSupervisorPageInfo");
                String topoName = common.topoName;
                Assignment assignment = common.assignment;
                Map<List<Integer>, Map<String, Object>> beats = common.beats;
                Map<Integer, String> taskToComp = common.taskToComponent;
                Map<List<Long>, List<Object>> exec2NodePort = new HashMap<>();
                Map<String, String> nodeToHost;
                if (assignment != null) {
                    Map<List<Long>, NodeInfo> execToNodeInfo = assignment.get_executor_node_port();
                    for (Entry<List<Long>, NodeInfo> entry : execToNodeInfo.entrySet()) {
                        NodeInfo ni = entry.getValue();
                        List<Object> nodePort = Arrays.asList(ni.get_node(), ni.get_port_iterator().next());
                        exec2NodePort.put(entry.getKey(), nodePort);
                    }
                    nodeToHost = assignment.get_node_host();
                } else {
                    nodeToHost = Collections.emptyMap();
                }
                Map<WorkerSlot, WorkerResources> workerResources = getWorkerResourcesForTopology(topoId);
                boolean isAllowed = userTopologies.contains(topoId);
                String owner = (common.base == null) ? null : common.base.get_owner();
                for (WorkerSummary workerSummary : StatsUtil.aggWorkerStats(topoId, topoName, taskToComp, beats, exec2NodePort, nodeToHost, workerResources, includeSys, isAllowed, sid, owner)) {
                    pageInfo.add_to_worker_summaries(workerSummary);
                }
            }
        }
        return pageInfo;
    } catch (Exception e) {
        LOG.warn("Get super page info exception. (super id='{}')", superId, e);
        if (e instanceof TException) {
            throw (TException) e;
        }
        throw new RuntimeException(e);
    }
}

Example 13 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class LocalAssignmentsBackendTest method mockedAssignment.

private Assignment mockedAssignment(int i) {
    Assignment ass = new Assignment();
    ass.set_master_code_dir("master_code_dir" + i);
    HashMap node_to_host = new HashMap();
    node_to_host.put("node" + i, "host" + i);
    ass.set_node_host(node_to_host);
    Map<List<Long>, NodeInfo> executor_node_port = new HashMap<>();
    Set<Long> nodePorts = new HashSet<>();
    nodePorts.add(9723L);
    executor_node_port.put(Arrays.asList(i + 0L), new NodeInfo("node" + i, nodePorts));
    ass.set_executor_node_port(executor_node_port);
    Map<List<Long>, Long> executor_start_time_secs = new HashMap<>();
    executor_start_time_secs.put(Arrays.asList(1L), 12345L);
    ass.set_executor_start_time_secs(executor_start_time_secs);
    ass.set_worker_resources(new HashedMap());
    ass.set_total_shared_off_heap(new HashedMap());
    ass.set_owner("o");
    return ass;
}

Example 14 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class LocalAssignmentsBackendTest method testLocalAssignment.

@Test
public void testLocalAssignment() {
    Map<String, Assignment> stormToAssignment = new HashMap<>();
    String storm1 = "storm1";
    String storm2 = "storm2";
    Assignment ass1 = mockedAssignment(1);
    Assignment ass2 = mockedAssignment(2);
    ILocalAssignmentsBackend backend = LocalAssignmentsBackendFactory.getBackend(ConfigUtils.readStormConfig());
    assertEquals(null, backend.getAssignment(storm1));
    backend.keepOrUpdateAssignment(storm1, ass1);
    backend.keepOrUpdateAssignment(storm2, ass2);
    assertEquals(ass1, backend.getAssignment(storm1));
    assertEquals(ass2, backend.getAssignment(storm2));
    backend.clearStateForStorm(storm1);
    assertEquals(null, backend.getAssignment(storm1));
    backend.keepOrUpdateAssignment(storm1, ass1);
    backend.keepOrUpdateAssignment(storm1, ass2);
    assertEquals(ass2, backend.getAssignment(storm1));
}

Example 15 with Assignment

use of org.apache.storm.generated.Assignment in project storm by apache.

the class WorkerState method refreshConnections.

public void refreshConnections(Runnable callback) throws Exception {
    Integer version = stormClusterState.assignmentVersion(topologyId, callback);
    version = (null == version) ? 0 : version;
    VersionedData<Assignment> assignmentVersion = assignmentVersions.get().get(topologyId);
    Assignment assignment;
    if (null != assignmentVersion && (assignmentVersion.getVersion() == version)) {
        assignment = assignmentVersion.getData();
    } else {
        VersionedData<Assignment> newAssignmentVersion = new VersionedData<>(version, stormClusterState.assignmentInfoWithVersion(topologyId, callback).getData());
        assignmentVersions.getAndUpdate(prev -> {
            Map<String, VersionedData<Assignment>> next = new HashMap<>(prev);
            next.put(topologyId, newAssignmentVersion);
            return next;
        });
        assignment = newAssignmentVersion.getData();
    }
    Set<NodeInfo> neededConnections = new HashSet<>();
    Map<Integer, NodeInfo> newTaskToNodePort = new HashMap<>();
    if (null != assignment) {
        Map<Integer, NodeInfo> taskToNodePort = StormCommon.taskToNodeport(assignment.get_executor_node_port());
        for (Map.Entry<Integer, NodeInfo> taskToNodePortEntry : taskToNodePort.entrySet()) {
            Integer task = taskToNodePortEntry.getKey();
            if (outboundTasks.contains(task)) {
                newTaskToNodePort.put(task, taskToNodePortEntry.getValue());
                if (!taskIds.contains(task)) {
                    neededConnections.add(taskToNodePortEntry.getValue());
                }
            }
        }
    }
    Set<NodeInfo> currentConnections = cachedNodeToPortSocket.get().keySet();
    Set<NodeInfo> newConnections = Sets.difference(neededConnections, currentConnections);
    Set<NodeInfo> removeConnections = Sets.difference(currentConnections, neededConnections);
    // Add new connections atomically
    cachedNodeToPortSocket.getAndUpdate(prev -> {
        Map<NodeInfo, IConnection> next = new HashMap<>(prev);
        for (NodeInfo nodeInfo : newConnections) {
            next.put(nodeInfo, mqContext.connect(topologyId, assignment.get_node_host().get(nodeInfo.get_node()), nodeInfo.get_port().iterator().next().intValue()));
        }
        return next;
    });
    try {
        endpointSocketLock.writeLock().lock();
        cachedTaskToNodePort.set(newTaskToNodePort);
    } finally {
        endpointSocketLock.writeLock().unlock();
    }
    for (NodeInfo nodeInfo : removeConnections) {
        cachedNodeToPortSocket.get().get(nodeInfo).close();
    }
    // Remove old connections atomically
    cachedNodeToPortSocket.getAndUpdate(prev -> {
        Map<NodeInfo, IConnection> next = new HashMap<>(prev);
        removeConnections.forEach(next::remove);
        return next;
    });
}