Examples with SystemConfig com.twitter.heron.common.config.SystemConfig used on opensource projects

Example 1 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class MetricsManager method main.

public static void main(String[] args) throws IOException {
    if (args.length != 6) {
        throw new RuntimeException("Invalid arguments; Usage: java com.twitter.heron.metricsmgr.MetricsManager " + "<id> <port> <topname> <topid> <heron_internals_config_filename> " + "<metrics_sinks_config_filename>");
    }
    String metricsmgrId = args[0];
    int metricsPort = Integer.parseInt(args[1]);
    String topologyName = args[2];
    String topologyId = args[3];
    String systemConfigFilename = args[4];
    String metricsSinksConfigFilename = args[5];
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).build();
    // Add the SystemConfig into SingletonRegistry
    SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
    // Init the logging setting and redirect the stdout and stderr to logging
    // For now we just set the logging level as INFO; later we may accept an argument to set it.
    Level loggingLevel = Level.INFO;
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    // Log to file and TMaster
    LoggingHelper.loggerInit(loggingLevel, true);
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(metricsmgrId, loggingDir, true, systemConfig.getHeronLoggingMaximumSizeMb() * Constants.MB_TO_BYTES, systemConfig.getHeronLoggingMaximumFiles()));
    LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
    LOG.info(String.format("Starting Metrics Manager for topology %s with topologyId %s with " + "Metrics Manager Id %s, Merics Manager Port: %d.", topologyName, topologyId, metricsmgrId, metricsPort));
    LOG.info("System Config: " + systemConfig);
    // Populate the config
    MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename);
    LOG.info("Sinks Config:" + sinksConfig.toString());
    MetricsManager metricsManager = new MetricsManager(topologyName, METRICS_MANAGER_HOST, metricsPort, metricsmgrId, systemConfig, sinksConfig);
    metricsManager.start();
    LOG.info("Loops terminated. Metrics Manager exits.");
}

Example 2 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class HeronInstance method main.

public static void main(String[] args) throws IOException {
    if (args.length < 10) {
        throw new RuntimeException("Invalid arguments; Usage is java com.twitter.heron.instance.HeronInstance " + "<topology_name> <topology_id> <instance_id> <component_name> <task_id> " + "<component_index> <stmgr_id> <stmgr_port> <metricsmgr_port> " + "<heron_internals_config_filename>");
    }
    String topologyName = args[0];
    String topologyId = args[1];
    String instanceId = args[2];
    String componentName = args[3];
    int taskId = Integer.parseInt(args[4]);
    int componentIndex = Integer.parseInt(args[5]);
    String streamId = args[6];
    int streamPort = Integer.parseInt(args[7]);
    int metricsPort = Integer.parseInt(args[8]);
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(args[9], true).build();
    // Add the SystemConfig into SingletonRegistry
    SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
    // Create the protobuf Instance
    PhysicalPlans.InstanceInfo instanceInfo = PhysicalPlans.InstanceInfo.newBuilder().setTaskId(taskId).setComponentIndex(componentIndex).setComponentName(componentName).build();
    PhysicalPlans.Instance instance = PhysicalPlans.Instance.newBuilder().setInstanceId(instanceId).setStmgrId(streamId).setInfo(instanceInfo).build();
    // Init the logging setting and redirect the stdout and stderr to logging
    // For now we just set the logging level as INFO; later we may accept an argument to set it.
    Level loggingLevel = Level.INFO;
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    // Log to file and TMaster
    LoggingHelper.loggerInit(loggingLevel, true);
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(instanceId, loggingDir, true, systemConfig.getHeronLoggingMaximumSizeMb() * Constants.MB_TO_BYTES, systemConfig.getHeronLoggingMaximumFiles()));
    LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
    LOG.info("\nStarting instance " + instanceId + " for topology " + topologyName + " and topologyId " + topologyId + " for component " + componentName + " with taskId " + taskId + " and componentIndex " + componentIndex + " and stmgrId " + streamId + " and stmgrPort " + streamPort + " and metricsManagerPort " + metricsPort);
    LOG.info("System Config: " + systemConfig);
    HeronInstance heronInstance = new HeronInstance(topologyName, topologyId, instance, streamPort, metricsPort);
    heronInstance.start();
}

Example 3 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class SpoutMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = systemConfig.getHeronMetricsExportIntervalSec();
    topologyContext.registerMetric("__ack-count/default", ackCount, interval);
    topologyContext.registerMetric("__complete-latency/default", completeLatency, interval);
    topologyContext.registerMetric("__fail-latency/default", failLatency, interval);
    topologyContext.registerMetric("__fail-count/default", failCount, interval);
    topologyContext.registerMetric("__timeout-count/default", timeoutCount, interval);
    topologyContext.registerMetric("__emit-count/default", emitCount, interval);
    topologyContext.registerMetric("__next-tuple-latency", nextTupleLatency, interval);
    topologyContext.registerMetric("__next-tuple-count", nextTupleCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
    topologyContext.registerMetric("__pending-acked-count", pendingTuplesCount, interval);
}

Example 4 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class JVMMetrics method registerMetrics.

/**
   * Register metrics with the metrics collector
   */
public void registerMetrics(MetricsCollector metricsCollector) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = systemConfig.getHeronMetricsExportIntervalSec();
    metricsCollector.registerMetric("__jvm-gc-collection-time-ms", jvmGCTimeMs, interval);
    metricsCollector.registerMetric("__jvm-gc-collection-count", jvmGCCount, interval);
    metricsCollector.registerMetric("__jvm-gc-time-ms", jvmGCTimeMsPerGCType, interval);
    metricsCollector.registerMetric("__jvm-gc-count", jvmGCCountPerGCType, interval);
    metricsCollector.registerMetric("__jvm-uptime-secs", jvmUpTimeSecs, interval);
    metricsCollector.registerMetric("__jvm-thread-count", jvmThreadCount, interval);
    metricsCollector.registerMetric("__jvm-daemon-thread-count", jvmDaemonThreadCount, interval);
    metricsCollector.registerMetric("__jvm-process-cpu-time-nanos", processCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-threads-cpu-time-nanos", threadsCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-other-threads-cpu-time-nanos", otherThreadsCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-threads-user-cpu-time-nanos", threadsUserCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-other-threads-user-cpu-time-nanos", otherThreadsUserCPUTimeNs, interval);
    metricsCollector.registerMetric("__jvm-process-cpu-load", processCPULoad, interval);
    metricsCollector.registerMetric("__jvm-fd-count", fdCount, interval);
    metricsCollector.registerMetric("__jvm-fd-limit", fdLimit, interval);
    metricsCollector.registerMetric("__jvm-memory-free-mb", jvmMemoryFreeMB, interval);
    metricsCollector.registerMetric("__jvm-memory-used-mb", jvmMemoryUsedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-mb-total", jvmMemoryTotalMB, interval);
    metricsCollector.registerMetric("__jvm-memory-heap-mb-used", jvmMemoryHeapUsedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-heap-mb-committed", jvmMemoryHeapCommittedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-heap-mb-max", jvmMemoryHeapMaxMB, interval);
    metricsCollector.registerMetric("__jvm-memory-non-heap-mb-used", jvmMemoryNonHeapUsedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-non-heap-mb-committed", jvmMemoryNonHeapCommittedMB, interval);
    metricsCollector.registerMetric("__jvm-memory-non-heap-mb-max", jvmMemoryNonHeapMaxMB, interval);
    metricsCollector.registerMetric("__jvm-peak-usage", jvmPeakUsagePerMemoryPool, interval);
    metricsCollector.registerMetric("__jvm-collection-usage", jvmCollectionUsagePerMemoryPool, interval);
    metricsCollector.registerMetric("__jvm-estimated-usage", jvmEstimatedUsagePerMemoryPool, interval);
    metricsCollector.registerMetric("__jvm-buffer-pool", jvmBufferPoolMemoryUsage, interval);
}

Example 5 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class GatewayMetrics method registerMetrics.

/**
   * Register default Gateway Metrics to given MetricsCollector
   *
   * @param metricsCollector the MetricsCollector to register Metrics on
   */
public void registerMetrics(MetricsCollector metricsCollector) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = systemConfig.getHeronMetricsExportIntervalSec();
    metricsCollector.registerMetric("__gateway-received-packets-size", receivedPacketsSize, interval);
    metricsCollector.registerMetric("__gateway-sent-packets-size", sentPacketsSize, interval);
    metricsCollector.registerMetric("__gateway-received-packets-count", receivedPacketsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-packets-count", sentPacketsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-metrics-size", sentMetricsSize, interval);
    metricsCollector.registerMetric("__gateway-sent-metrics-packets-count", sentMetricsPacketsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-metrics-count", sentMetricsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-exceptions-count", sentExceptionsCount, interval);
    metricsCollector.registerMetric("__gateway-in-stream-queue-size", inStreamQueueSize, interval);
    metricsCollector.registerMetric("__gateway-out-stream-queue-size", outStreamQueueSize, interval);
    metricsCollector.registerMetric("__gateway-in-stream-queue-expected-capacity", inStreamQueueExpectedCapacity, interval);
    metricsCollector.registerMetric("__gateway-out-stream-queue-expected-capacity", outStreamQueueExpectedCapacity, interval);
    metricsCollector.registerMetric("__gateway-in-queue-full-count", inQueueFullCount, interval);
}