Examples with ClusterMetricsConsumerExecutor org.apache.storm.metric.ClusterMetricsConsumerExecutor used on opensource projects

Example 1 with ClusterMetricsConsumerExecutor

use of org.apache.storm.metric.ClusterMetricsConsumerExecutor in project storm by apache.

the class Nimbus method makeClusterMetricsConsumerExecutors.

@SuppressWarnings("unchecked")
private static List<ClusterMetricsConsumerExecutor> makeClusterMetricsConsumerExecutors(Map<String, Object> conf) {
    Collection<Map<String, Object>> consumers = (Collection<Map<String, Object>>) conf.get(DaemonConfig.STORM_CLUSTER_METRICS_CONSUMER_REGISTER);
    List<ClusterMetricsConsumerExecutor> ret = new ArrayList<>();
    if (consumers != null) {
        for (Map<String, Object> consumer : consumers) {
            ret.add(new ClusterMetricsConsumerExecutor((String) consumer.get("class"), consumer.get("argument")));
        }
    }
    return ret;
}

Example 2 with ClusterMetricsConsumerExecutor

use of org.apache.storm.metric.ClusterMetricsConsumerExecutor in project storm by apache.

the class Nimbus method launchServer.

@VisibleForTesting
public void launchServer() throws Exception {
    try {
        IStormClusterState state = stormClusterState;
        NimbusInfo hpi = nimbusHostPortInfo;
        LOG.info("Starting Nimbus with conf {}", ConfigUtils.maskPasswords(conf));
        validator.prepare(conf);
        // add to nimbuses
        state.addNimbusHost(hpi.getHost(), new NimbusSummary(hpi.getHost(), hpi.getPort(), Time.currentTimeSecs(), false, STORM_VERSION));
        leaderElector.addToLeaderLockQueue();
        this.blobStore.startSyncBlobs();
        for (ClusterMetricsConsumerExecutor exec : clusterConsumerExceutors) {
            exec.prepare();
        }
        // Leadership coordination may be incomplete when launchServer is called. Previous behavior did a one time check
        // which could cause Nimbus to not process TopologyActions.GAIN_LEADERSHIP transitions. Similar problem exists for
        // HA Nimbus on being newly elected as leader. Change to a recurring pattern addresses these problems.
        timer.scheduleRecurring(3, 5, () -> {
            try {
                boolean isLeader = isLeader();
                if (isLeader && !wasLeader) {
                    for (String topoId : state.activeStorms()) {
                        transition(topoId, TopologyActions.GAIN_LEADERSHIP, null);
                    }
                    clusterMetricSet.setActive(true);
                }
                wasLeader = isLeader;
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        final boolean doNotReassign = (Boolean) conf.getOrDefault(ServerConfigUtils.NIMBUS_DO_NOT_REASSIGN, false);
        timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_MONITOR_FREQ_SECS)), () -> {
            try {
                if (!doNotReassign) {
                    mkAssignments();
                }
                doCleanup();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Schedule Nimbus inbox cleaner
        final int jarExpSecs = ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_INBOX_JAR_EXPIRATION_SECS));
        timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_CLEANUP_INBOX_FREQ_SECS)), () -> {
            try {
                cleanInbox(getInbox(), jarExpSecs);
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Schedule topology history cleaner
        Integer interval = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_CLEANUP_INTERVAL_SECS), null);
        if (interval != null) {
            final int lvCleanupAgeMins = ObjectReader.getInt(conf.get(DaemonConfig.LOGVIEWER_CLEANUP_AGE_MINS));
            timer.scheduleRecurring(0, interval, () -> {
                try {
                    cleanTopologyHistory(lvCleanupAgeMins);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            });
        }
        timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.NIMBUS_CREDENTIAL_RENEW_FREQ_SECS)), () -> {
            try {
                renewCredentials();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        // Periodically make sure the blobstore update time is up to date.  This could have failed if Nimbus encountered
        // an exception updating the update time, or due to bugs causing a missed update of the blobstore mod time on a blob
        // update.
        timer.scheduleRecurring(30, ServerConfigUtils.getLocalizerUpdateBlobInterval(conf) * 5, () -> {
            try {
                blobStore.validateBlobUpdateTime();
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        });
        metricsRegistry.registerGauge("nimbus:total-available-memory-non-negative", () -> nodeIdToResources.get().values().parallelStream().mapToDouble(supervisorResources -> Math.max(supervisorResources.getAvailableMem(), 0)).sum());
        metricsRegistry.registerGauge("nimbus:available-cpu-non-negative", () -> nodeIdToResources.get().values().parallelStream().mapToDouble(supervisorResources -> Math.max(supervisorResources.getAvailableCpu(), 0)).sum());
        metricsRegistry.registerGauge("nimbus:total-memory", () -> nodeIdToResources.get().values().parallelStream().mapToDouble(SupervisorResources::getTotalMem).sum());
        metricsRegistry.registerGauge("nimbus:total-cpu", () -> nodeIdToResources.get().values().parallelStream().mapToDouble(SupervisorResources::getTotalCpu).sum());
        metricsRegistry.registerGauge("nimbus:longest-scheduling-time-ms", () -> {
            // We want to update longest scheduling time in real time in case scheduler get stuck
            // Get current time before startTime to avoid potential race with scheduler's Timer
            Long currTime = Time.nanoTime();
            Long startTime = schedulingStartTimeNs.get();
            return TimeUnit.NANOSECONDS.toMillis(startTime == null ? longestSchedulingTime.get() : Math.max(currTime - startTime, longestSchedulingTime.get()));
        });
        metricsRegistry.registerMeter("nimbus:num-launched").mark();
        timer.scheduleRecurring(0, ObjectReader.getInt(conf.get(DaemonConfig.STORM_CLUSTER_METRICS_CONSUMER_PUBLISH_INTERVAL_SECS)), () -> {
            try {
                if (isLeader()) {
                    sendClusterMetricsToExecutors();
                }
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        timer.scheduleRecurring(5, 5, clusterMetricSet);
    } catch (Exception e) {
        if (Utils.exceptionCauseIsInstanceOf(InterruptedException.class, e)) {
            throw e;
        }
        if (Utils.exceptionCauseIsInstanceOf(InterruptedIOException.class, e)) {
            throw e;
        }
        LOG.error("Error on initialization of nimbus", e);
        Utils.exitProcess(13, "Error on initialization of nimbus");
    }
}

Example 3 with ClusterMetricsConsumerExecutor

use of org.apache.storm.metric.ClusterMetricsConsumerExecutor in project storm by apache.

the class Nimbus method sendClusterMetricsToExecutors.

private void sendClusterMetricsToExecutors() throws Exception {
    ClusterInfo clusterInfo = mkClusterInfo();
    ClusterSummary clusterSummary = getClusterInfoImpl();
    List<DataPoint> clusterMetrics = extractClusterMetrics(clusterSummary);
    Map<IClusterMetricsConsumer.SupervisorInfo, List<DataPoint>> supervisorMetrics = extractSupervisorMetrics(clusterSummary);
    for (ClusterMetricsConsumerExecutor consumerExecutor : clusterConsumerExceutors) {
        consumerExecutor.handleDataPoints(clusterInfo, clusterMetrics);
        for (Entry<IClusterMetricsConsumer.SupervisorInfo, List<DataPoint>> entry : supervisorMetrics.entrySet()) {
            consumerExecutor.handleDataPoints(entry.getKey(), entry.getValue());
        }
    }
}