Examples with NimbusSummary org.apache.storm.generated.NimbusSummary used on opensource projects

Example 1 with NimbusSummary

use of org.apache.storm.generated.NimbusSummary in project storm by apache.

the class Nimbus method getClusterInfoImpl.

private ClusterSummary getClusterInfoImpl() throws Exception {
    IStormClusterState state = stormClusterState;
    Map<String, SupervisorInfo> infos = state.allSupervisorInfo();
    List<SupervisorSummary> summaries = new ArrayList<>(infos.size());
    for (Entry<String, SupervisorInfo> entry : infos.entrySet()) {
        summaries.add(makeSupervisorSummary(entry.getKey(), entry.getValue()));
    }
    int uptime = this.uptime.upTime();
    Map<String, StormBase> bases = state.topologyBases();
    List<NimbusSummary> nimbuses = state.nimbuses();
    //update the isLeader field for each nimbus summary
    NimbusInfo leader = leaderElector.getLeader();
    for (NimbusSummary nimbusSummary : nimbuses) {
        nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs()));
        nimbusSummary.set_isLeader(leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port());
    }
    List<TopologySummary> topologySummaries = new ArrayList<>();
    for (Entry<String, StormBase> entry : bases.entrySet()) {
        StormBase base = entry.getValue();
        if (base == null) {
            continue;
        }
        String topoId = entry.getKey();
        Assignment assignment = state.assignmentInfo(topoId, null);
        int numTasks = 0;
        int numExecutors = 0;
        int numWorkers = 0;
        if (assignment != null && assignment.is_set_executor_node_port()) {
            for (List<Long> ids : assignment.get_executor_node_port().keySet()) {
                numTasks += StormCommon.executorIdToTasks(ids).size();
            }
            numExecutors = assignment.get_executor_node_port_size();
            numWorkers = new HashSet<>(assignment.get_executor_node_port().values()).size();
        }
        TopologySummary summary = new TopologySummary(topoId, base.get_name(), numTasks, numExecutors, numWorkers, Time.deltaSecs(base.get_launch_time_secs()), extractStatusStr(base));
        if (base.is_set_owner()) {
            summary.set_owner(base.get_owner());
        }
        String status = idToSchedStatus.get().get(topoId);
        if (status != null) {
            summary.set_sched_status(status);
        }
        TopologyResources resources = getResourcesForTopology(topoId, base);
        if (resources != null) {
            summary.set_requested_memonheap(resources.getRequestedMemOnHeap());
            summary.set_requested_memoffheap(resources.getRequestedMemOffHeap());
            summary.set_requested_cpu(resources.getRequestedCpu());
            summary.set_assigned_memonheap(resources.getAssignedMemOnHeap());
            summary.set_assigned_memoffheap(resources.getAssignedMemOffHeap());
            summary.set_assigned_cpu(resources.getAssignedCpu());
        }
        summary.set_replication_count(getBlobReplicationCount(ConfigUtils.masterStormCodeKey(topoId)));
        topologySummaries.add(summary);
    }
    ClusterSummary ret = new ClusterSummary(summaries, topologySummaries, nimbuses);
    ret.set_nimbus_uptime_secs(uptime);
    return ret;
}

Example 2 with NimbusSummary

use of org.apache.storm.generated.NimbusSummary in project storm by apache.

the class NimbusClient method getConfiguredClientAs.

public static NimbusClient getConfiguredClientAs(Map conf, String asUser) {
    if (conf.containsKey(Config.STORM_DO_AS_USER)) {
        if (asUser != null && !asUser.isEmpty()) {
            LOG.warn("You have specified a doAsUser as param {} and a doAsParam as config, config will take precedence.", asUser, conf.get(Config.STORM_DO_AS_USER));
        }
        asUser = (String) conf.get(Config.STORM_DO_AS_USER);
    }
    List<String> seeds;
    if (conf.containsKey(Config.NIMBUS_HOST)) {
        LOG.warn("Using deprecated config {} for backward compatibility. Please update your storm.yaml so it only has config {}", Config.NIMBUS_HOST, Config.NIMBUS_SEEDS);
        seeds = Lists.newArrayList(conf.get(Config.NIMBUS_HOST).toString());
    } else {
        seeds = (List<String>) conf.get(Config.NIMBUS_SEEDS);
    }
    for (String host : seeds) {
        int port = Integer.parseInt(conf.get(Config.NIMBUS_THRIFT_PORT).toString());
        NimbusSummary nimbusSummary;
        NimbusClient client = null;
        try {
            client = new NimbusClient(conf, host, port, null, asUser);
            nimbusSummary = client.getClient().getLeader();
            if (nimbusSummary != null) {
                String leaderNimbus = nimbusSummary.get_host() + ":" + nimbusSummary.get_port();
                LOG.info("Found leader nimbus : {}", leaderNimbus);
                if (nimbusSummary.get_host().equals(host) && nimbusSummary.get_port() == port) {
                    NimbusClient ret = client;
                    client = null;
                    return ret;
                }
                try {
                    return new NimbusClient(conf, nimbusSummary.get_host(), nimbusSummary.get_port(), null, asUser);
                } catch (TTransportException e) {
                    throw new RuntimeException("Failed to create a nimbus client for the leader " + leaderNimbus, e);
                }
            }
        } catch (Exception e) {
            LOG.warn("Ignoring exception while trying to get leader nimbus info from " + host + ". will retry with a different seed host.", e);
            continue;
        } finally {
            if (client != null) {
                client.close();
            }
        }
        throw new NimbusLeaderNotFoundException("Could not find a nimbus leader, please try " + "again after some time.");
    }
    throw new NimbusLeaderNotFoundException("Could not find leader nimbus from seed hosts " + seeds + ". " + "Did you specify a valid list of nimbus hosts for config " + Config.NIMBUS_SEEDS + "?");
}

Example 3 with NimbusSummary

use of org.apache.storm.generated.NimbusSummary in project storm by apache.

the class Nimbus method launchServer.

@VisibleForTesting
public void launchServer() throws Exception {
    try {
        BlobStore store = blobStore;
        IStormClusterState state = stormClusterState;
        NimbusInfo hpi = nimbusHostPortInfo;
        LOG.info("Starting Nimbus with conf {}", conf);
        validator.prepare(conf);
        //add to nimbuses
        state.addNimbusHost(hpi.getHost(), new NimbusSummary(hpi.getHost(), hpi.getPort(), Time.currentTimeSecs(), false, STORM_VERSION));
        leaderElector.addToLeaderLockQueue();
        if (store instanceof LocalFsBlobStore) {
            //register call back for blob-store
            state.blobstore(() -> {
                try {
                    blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
            setupBlobstore();
        }
        for (ClusterMetricsConsumerExecutor exec : clusterConsumerExceutors) {
            exec.prepare();
        }
        if (isLeader()) {
            for (String topoId : state.activeStorms()) {
                transition(topoId, TopologyActions.STARTUP, null);
            }
        }
        final boolean doNotReassign = (Boolean) conf.getOrDefault(ConfigUtils.NIMBUS_DO_NOT_REASSIGN, false);
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_MONITOR_FREQ_SECS)), () -> {
            try {
                if (!doNotReassign) {
                    mkAssignments();
                }
                doCleanup();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Schedule Nimbus inbox cleaner
        final int jarExpSecs = Utils.getInt(conf.get(Config.NIMBUS_INBOX_JAR_EXPIRATION_SECS));
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CLEANUP_INBOX_FREQ_SECS)), () -> {
            try {
                cleanInbox(getInbox(), jarExpSecs);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        //Schedule nimbus code sync thread to sync code from other nimbuses.
        if (store instanceof LocalFsBlobStore) {
            timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CODE_SYNC_FREQ_SECS)), () -> {
                try {
                    blobSync();
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
        // Schedule topology history cleaner
        Integer interval = Utils.getInt(conf.get(Config.LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
        if (interval != null) {
            final int lvCleanupAgeMins = Utils.getInt(conf.get(Config.LOGVIEWER_CLEANUP_AGE_MINS));
            timer.scheduleRecurring(0, interval, () -> {
                try {
                    cleanTopologyHistory(lvCleanupAgeMins);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
        timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.NIMBUS_CREDENTIAL_RENEW_FREQ_SECS)), () -> {
            try {
                renewCredentials();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        StormMetricsRegistry.registerGauge("nimbus:num-supervisors", () -> state.supervisors(null));
        StormMetricsRegistry.startMetricsReporters(conf);
        if (clusterConsumerExceutors != null) {
            timer.scheduleRecurring(0, Utils.getInt(conf.get(Config.STORM_CLUSTER_METRICS_CONSUMER_PUBLISH_INTERVAL_SECS)), () -> {
                try {
                    if (isLeader()) {
                        sendClusterMetricsToExecutors();
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
    } catch (Exception e) {
        if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e)) {
            throw e;
        }
        if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, e)) {
            throw e;
        }
        LOG.error("Error on initialization of nimbus", e);
        Utils.exitProcess(13, "Error on initialization of nimbus");
    }
}

Example 4 with NimbusSummary

use of org.apache.storm.generated.NimbusSummary in project storm by apache.

the class Nimbus method getLeader.

@Override
public NimbusSummary getLeader() throws AuthorizationException, TException {
    getLeaderCalls.mark();
    checkAuthorization(null, null, "getClusterInfo");
    List<NimbusSummary> nimbuses = stormClusterState.nimbuses();
    NimbusInfo leader = leaderElector.getLeader();
    for (NimbusSummary nimbusSummary : nimbuses) {
        if (leader.getHost().equals(nimbusSummary.get_host()) && leader.getPort() == nimbusSummary.get_port()) {
            nimbusSummary.set_uptime_secs(Time.deltaSecs(nimbusSummary.get_uptime_secs()));
            nimbusSummary.set_isLeader(true);
            return nimbusSummary;
        }
    }
    return null;
}