Examples with SupervisorAssignments org.apache.storm.generated.SupervisorAssignments used on opensource projects

Example 1 with SupervisorAssignments

use of org.apache.storm.generated.SupervisorAssignments in project storm by apache.

the class Nimbus method notifySupervisorsAssignments.

/**
 * Notify supervisors/nodes assigned assignments.
 *
 * @param assignments       assignments map for nodes
 * @param service           {@link AssignmentDistributionService} for distributing assignments asynchronous
 * @param nodeHost          node -> host map
 * @param supervisorDetails nodeId -> {@link SupervisorDetails} map
 */
private static void notifySupervisorsAssignments(Map<String, Assignment> assignments, AssignmentDistributionService service, Map<String, String> nodeHost, Map<String, SupervisorDetails> supervisorDetails, StormMetricsRegistry metricsRegistry) {
    for (Map.Entry<String, String> nodeEntry : nodeHost.entrySet()) {
        try {
            String nodeId = nodeEntry.getKey();
            String hostname = nodeEntry.getValue();
            SupervisorAssignments supervisorAssignments = new SupervisorAssignments();
            supervisorAssignments.set_storm_assignment(assignmentsForHost(assignments, hostname));
            SupervisorDetails details = supervisorDetails.get(nodeId);
            Integer serverPort = details != null ? details.getServerPort() : null;
            service.addAssignmentsForNode(nodeId, nodeEntry.getValue(), serverPort, supervisorAssignments, metricsRegistry);
        } catch (Throwable tr1) {
            // just skip when any error happens wait for next round assignments reassign
            LOG.error("Exception when add assignments distribution task for node {}", nodeEntry.getKey());
        }
    }
}

Example 2 with SupervisorAssignments

use of org.apache.storm.generated.SupervisorAssignments in project storm by apache.

the class Supervisor method createSupervisorIface.

private org.apache.storm.generated.Supervisor.Iface createSupervisorIface() {
    return new org.apache.storm.generated.Supervisor.Iface() {

        @Override
        public void sendSupervisorAssignments(SupervisorAssignments assignments) throws AuthorizationException, TException {
            checkAuthorization("sendSupervisorAssignments");
            LOG.info("Got an assignments from master, will start to sync with assignments: {}", assignments);
            SynchronizeAssignments syn = new SynchronizeAssignments(getSupervisor(), assignments, getReadClusterState());
            getEventManger().add(syn);
        }

        @Override
        public Assignment getLocalAssignmentForStorm(String id) throws NotAliveException, AuthorizationException, TException {
            Map<String, Object> topoConf = null;
            try {
                topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
            } catch (IOException e) {
                LOG.warn("Topology config is not localized yet...");
            }
            checkAuthorization(id, topoConf, "getLocalAssignmentForStorm");
            Assignment assignment = getStormClusterState().assignmentInfo(id, null);
            if (null == assignment) {
                throw new WrappedNotAliveException("No local assignment assigned for storm: " + id + " for node: " + getHostName());
            }
            return assignment;
        }

        @Override
        public void sendSupervisorWorkerHeartbeat(SupervisorWorkerHeartbeat heartbeat) throws AuthorizationException, NotAliveException, TException {
            // do nothing except validate heartbeat for now.
            String id = heartbeat.get_storm_id();
            Map<String, Object> topoConf = null;
            try {
                topoConf = ConfigUtils.readSupervisorStormConf(conf, id);
            } catch (IOException e) {
                LOG.warn("Topology config is not localized yet...");
                throw new WrappedNotAliveException(id + " does not appear to be alive, you should probably exit");
            }
            checkAuthorization(id, topoConf, "sendSupervisorWorkerHeartbeat");
        }
    };
}

Example 3 with SupervisorAssignments

use of org.apache.storm.generated.SupervisorAssignments in project storm by apache.

the class SynchronizeAssignments method getAssignmentsFromMaster.

/**
 * Used by {@link Supervisor} to fetch assignments when start up.
 * @param conf config
 * @param clusterState {@link IStormClusterState}
 * @param node id of node
 */
public void getAssignmentsFromMaster(Map conf, IStormClusterState clusterState, String node) {
    if (ConfigUtils.isLocalMode(conf)) {
        try {
            List<SupervisorAssignments> supervisorAssignmentsList = getAllAssignmentsFromNumaSupervisors(this.supervisor.getLocalNimbus(), node);
            assignedAssignmentsToLocal(clusterState, supervisorAssignmentsList);
        } catch (TException e) {
            LOG.error("Get assignments from local master exception", e);
        }
    } else {
        try (NimbusClient master = NimbusClient.getConfiguredClient(conf)) {
            List<SupervisorAssignments> supervisorAssignmentsList = getAllAssignmentsFromNumaSupervisors(master.getClient(), node);
            LOG.debug("Sync an assignments from master, will start to sync with assignments: {}", supervisorAssignmentsList);
            assignedAssignmentsToLocal(clusterState, supervisorAssignmentsList);
        } catch (Exception t) {
            LOG.error("Get assignments from master exception", t);
        }
    }
}

Example 4 with SupervisorAssignments

use of org.apache.storm.generated.SupervisorAssignments in project storm by apache.

the class SynchronizeAssignments method assignedAssignmentsToLocal.

private static void assignedAssignmentsToLocal(IStormClusterState clusterState, List<SupervisorAssignments> supervisorAssignments) {
    if (null == supervisorAssignments || supervisorAssignments.isEmpty()) {
        // unknown error, just skip
        return;
    }
    Map<String, byte[]> serAssignments = new HashMap<>();
    for (SupervisorAssignments supervisorAssignment : supervisorAssignments) {
        if (supervisorAssignment == null) {
            // unknown error, just skip
            continue;
        }
        for (Map.Entry<String, Assignment> entry : supervisorAssignment.get_storm_assignment().entrySet()) {
            serAssignments.put(entry.getKey(), Utils.serialize(entry.getValue()));
        }
    }
    clusterState.syncRemoteAssignments(serAssignments);
}

Example 5 with SupervisorAssignments

use of org.apache.storm.generated.SupervisorAssignments in project storm by apache.

the class SynchronizeAssignments method getAllAssignmentsFromNumaSupervisors.

public List<SupervisorAssignments> getAllAssignmentsFromNumaSupervisors(Nimbus.Iface nimbus, String node) throws TException {
    List<SupervisorAssignments> supervisorAssignmentsList = new ArrayList();
    Map<String, Object> validatedNumaMap = SupervisorUtils.getNumaMap(supervisor.getConf());
    for (Map.Entry<String, Object> numaEntry : validatedNumaMap.entrySet()) {
        String numaId = numaEntry.getKey();
        SupervisorAssignments assignments = nimbus.getSupervisorAssignments(node + ServerConstants.NUMA_ID_SEPARATOR + numaId);
        supervisorAssignmentsList.add(assignments);
    }
    SupervisorAssignments assignments = nimbus.getSupervisorAssignments(node);
    supervisorAssignmentsList.add(assignments);
    return supervisorAssignmentsList;
}