Examples with SystemConfig org.apache.heron.common.config.SystemConfig used on opensource projects

Example 1 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class MetricsManager method main.

public static void main(String[] args) throws Exception {
    final Options options = constructOptions();
    final Options helpOptions = constructHelpOptions();
    final CommandLineParser parser = new DefaultParser();
    // parse the help options first.
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(options);
        return;
    }
    try {
        // Now parse the required options
        cmd = parser.parse(options, args);
    } catch (ParseException pe) {
        usage(options);
        throw new RuntimeException("Error parsing command line options: ", pe);
    }
    String metricsmgrId = cmd.getOptionValue("id");
    int metricsPort = Integer.parseInt(cmd.getOptionValue("port"));
    String topologyName = cmd.getOptionValue("topology");
    String topologyId = cmd.getOptionValue("topology-id");
    String systemConfigFilename = cmd.getOptionValue("system-config-file");
    String overrideConfigFilename = cmd.getOptionValue("override-config-file");
    String metricsSinksConfigFilename = cmd.getOptionValue("sink-config-file");
    String cluster = cmd.getOptionValue("cluster");
    String role = cmd.getOptionValue("role");
    String environment = cmd.getOptionValue("environment");
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).putAll(overrideConfigFilename, true).build();
    // Add the SystemConfig into SingletonRegistry
    SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
    // Init the logging setting and redirect the stdout and stderr to logging
    // For now we just set the logging level as INFO; later we may accept an argument to set it.
    Level loggingLevel = Level.INFO;
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    // Log to file and TManager
    LoggingHelper.loggerInit(loggingLevel, true);
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(metricsmgrId, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
    LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
    LOG.info(String.format("Starting Metrics Manager for topology %s with topologyId %s with " + "Metrics Manager Id %s, Metrics Manager Port: %d, for cluster/role/env %s.", topologyName, topologyId, metricsmgrId, metricsPort, String.format("%s/%s/%s", cluster, role, environment)));
    LOG.info("System Config: " + systemConfig);
    // Populate the config
    MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename, overrideConfigFilename);
    LOG.info("Sinks Config:" + sinksConfig.toString());
    MetricsManager metricsManager = new MetricsManager(topologyName, cluster, role, environment, METRICS_MANAGER_HOST, metricsPort, metricsmgrId, systemConfig, sinksConfig);
    metricsManager.start();
    LOG.info("Loops terminated. Metrics Manager exits.");
}

Example 2 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class SchedulerMain method setupLogging.

// Set up logging based on the Config
private static void setupLogging(Config config) throws IOException {
    String systemConfigFilename = Context.systemConfigFile(config);
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).build();
    // Init the logging setting and redirect the stdout and stderr to logging
    // For now we just set the logging level as INFO; later we may accept an argument to set it.
    Level loggingLevel = Level.INFO;
    if (Context.verbose(config).booleanValue()) {
        loggingLevel = Level.FINE;
    }
    // TODO(mfu): The folder creation may be duplicated with heron-executor in future
    // TODO(mfu): Remove the creation in future if feasible
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    if (!FileUtils.isDirectoryExists(loggingDir)) {
        FileUtils.createDirectory(loggingDir);
    }
    // Log to file
    LoggingHelper.loggerInit(loggingLevel, true);
    // TODO(mfu): Pass the scheduler id from cmd
    String processId = String.format("%s-%s-%s", "heron", Context.topologyName(config), "scheduler");
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(processId, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
    LOG.info("Logging setup done.");
}

Example 3 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class FullBoltMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = (int) systemConfig.getHeronMetricsExportInterval().getSeconds();
    topologyContext.registerMetric("__ack-count", ackCount, interval);
    topologyContext.registerMetric("__process-latency", processLatency, interval);
    topologyContext.registerMetric("__fail-latency", failLatency, interval);
    topologyContext.registerMetric("__fail-count", failCount, interval);
    topologyContext.registerMetric("__execute-count", executeCount, interval);
    topologyContext.registerMetric("__execute-latency", executeLatency, interval);
    topologyContext.registerMetric("__execute-time-ns", executeTimeNs, interval);
    topologyContext.registerMetric("__emit-count", emitCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
    topologyContext.registerMetric("__task-run-count", taskRunCount, interval);
    topologyContext.registerMetric("__execution-count", executionCount, interval);
    topologyContext.registerMetric("__continue-work-count", continueWorkCount, interval);
    topologyContext.registerMetric("__tuple-deserialization-time-ns", totalDeserializationTimeNs, interval);
    topologyContext.registerMetric("__tuple-serialization-time-ns", totalSerializationTimeNs, interval);
    topologyContext.registerMetric("__av-tuple-deserialization-time-ns", totalDeserializationTimeNs, interval);
    topologyContext.registerMetric("__av-tuple-serialization-time-ns", totalSerializationTimeNs, interval);
    topologyContext.registerMetric("__data-tuple-added-to-outgoing-queue/default", tupleAddedToQueue, interval);
}

Example 4 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class JVMMetrics method registerMetrics.

/**
 * Register metrics with the metrics collector
 */
public void registerMetrics(MetricsCollector metricsCollector) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = (int) systemConfig.getHeronMetricsExportInterval().getSeconds();
    metricsCollector.registerMetric("__jvm-gc-collection-time-ms", jvmGCTimeMs, interval);
    metricsCollector.registerMetric("__jvm-gc-collection-count", jvmGCCount, interval);
    metricsCollector.registerMetric("__jvm-gc-time-ms", jvmGCTimeMsPerGCType, interval);
    metricsCollector.registerMetric("__jvm-gc-count", jvmGCCountPerGCType, interval);
    metricsCollector.registerMetric("__jvm-uptime-secs", jvmUpTimeSecs, interval);
    metricsCollector.registerMetric("__jvm-thread-count", jvmThreadCount, interval);
    metricsCollector.registerMetric("__jvm-daemon-thread-count", jvmDaemonThreadCount, interval);
    metricsCollector.registerMetric("__jvm-process-cpu-time-nanos", processCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-threads-cpu-time-nanos", threadsCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-other-threads-cpu-time-nanos", otherThreadsCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-threads-user-cpu-time-nanos", threadsUserCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-other-threads-user-cpu-time-nanos", otherThreadsUserCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-process-cpu-load", processCPULoad, interval);
    metricsCollector.registerMetric("__jvm-fd-count", fdCount, interval);
    metricsCollector.registerMetric("__jvm-fd-limit", fdLimit, interval);
    metricsCollector.registerMetric("__jvm-memory-free-mb", jvmMemoryFreeMB, interval);
    metricsCollector.registerMetric("__jvm-memory-used-mb", jvmMemoryUsedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-mb-total", jvmMemoryTotalMB, interval);
    metricsCollector.registerMetric("__jvm-memory-heap-mb-used", jvmMemoryHeapUsedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-heap-mb-committed", jvmMemoryHeapCommittedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-heap-mb-max", jvmMemoryHeapMaxMB, interval);
    metricsCollector.registerMetric("__jvm-memory-non-heap-mb-used", jvmMemoryNonHeapUsedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-non-heap-mb-committed", jvmMemoryNonHeapCommittedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-non-heap-mb-max", jvmMemoryNonHeapMaxMB, interval);
    metricsCollector.registerMetric("__jvm-peak-usage", jvmPeakUsagePerMemoryPool, interval);
    metricsCollector.registerMetric("__jvm-collection-usage", jvmCollectionUsagePerMemoryPool, interval);
    metricsCollector.registerMetric("__jvm-estimated-usage", jvmEstimatedUsagePerMemoryPool, interval);
    metricsCollector.registerMetric("__jvm-buffer-pool", jvmBufferPoolMemoryUsage, interval);
}

Example 5 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class SpoutMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = (int) systemConfig.getHeronMetricsExportInterval().getSeconds();
    topologyContext.registerMetric("__ack-count/default", ackCount, interval);
    topologyContext.registerMetric("__complete-latency/default", completeLatency, interval);
    topologyContext.registerMetric("__fail-latency/default", failLatency, interval);
    topologyContext.registerMetric("__fail-count/default", failCount, interval);
    topologyContext.registerMetric("__timeout-count/default", timeoutCount, interval);
    topologyContext.registerMetric("__emit-count/default", emitCount, interval);
    topologyContext.registerMetric("__next-tuple-latency", nextTupleLatency, interval);
    topologyContext.registerMetric("__next-tuple-count", nextTupleCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
    topologyContext.registerMetric("__pending-acked-count", pendingTuplesCount, interval);
    topologyContext.registerMetric("__data-tuple-added-to-outgoing-queue/default", tupleAddedToQueue, interval);
    topologyContext.registerMetric("__task-run-count", taskRunCount, interval);
    topologyContext.registerMetric("__produce-tuple-count", produceTupleCount, interval);
    topologyContext.registerMetric("__continue-work-count", continueWorkCount, interval);
}