Examples with SystemConfig org.apache.heron.common.config.SystemConfig used on opensource projects

Example 6 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class BoltMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = (int) systemConfig.getHeronMetricsExportInterval().getSeconds();
    topologyContext.registerMetric("__ack-count/default", ackCount, interval);
    topologyContext.registerMetric("__process-latency/default", processLatency, interval);
    topologyContext.registerMetric("__fail-latency/default", failLatency, interval);
    topologyContext.registerMetric("__fail-count/default", failCount, interval);
    topologyContext.registerMetric("__execute-count/default", executeCount, interval);
    topologyContext.registerMetric("__execute-latency/default", executeLatency, interval);
    topologyContext.registerMetric("__emit-count/default", emitCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
    topologyContext.registerMetric("__data-tuple-added-to-outgoing-queue/default", tupleAddedToQueue, interval);
    topologyContext.registerMetric("__task-run-count", taskRunCount, interval);
    topologyContext.registerMetric("__execution-count", executionCount, interval);
    topologyContext.registerMetric("__continue-work-count", continueWorkCount, interval);
}

Example 7 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class HeronInstance method main.

public static void main(String[] args) throws IOException {
    CommandLine commandLine = parseCommandLineArgs(args);
    String topologyName = commandLine.getOptionValue(CommandLineOptions.TOPOLOGY_NAME_OPTION);
    String topologyId = commandLine.getOptionValue(CommandLineOptions.TOPOLOGY_ID_OPTION);
    String instanceId = commandLine.getOptionValue(CommandLineOptions.INSTANCE_ID_OPTION);
    String componentName = commandLine.getOptionValue(CommandLineOptions.COMPONENT_NAME_OPTION);
    Integer taskId = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.TASK_ID_OPTION));
    Integer componentIndex = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.COMPONENT_INDEX_OPTION));
    String streamId = commandLine.getOptionValue(CommandLineOptions.STMGR_ID_OPTION);
    Integer streamPort = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.STMGR_PORT_OPTION));
    Integer metricsPort = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.METRICS_MGR_PORT_OPTION));
    String systemConfigFile = commandLine.getOptionValue(CommandLineOptions.SYSTEM_CONFIG_FILE);
    String overrideConfigFile = commandLine.getOptionValue(CommandLineOptions.OVERRIDE_CONFIG_FILE);
    Integer remoteDebuggerPort = null;
    if (commandLine.hasOption(CommandLineOptions.REMOTE_DEBUGGER_PORT)) {
        remoteDebuggerPort = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.REMOTE_DEBUGGER_PORT));
    }
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFile, true).putAll(overrideConfigFile, true).build();
    // Add the SystemConfig into SingletonRegistry
    SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
    // Create the protobuf Instance
    PhysicalPlans.InstanceInfo.Builder instanceInfoBuilder = PhysicalPlans.InstanceInfo.newBuilder().setTaskId(taskId).setComponentIndex(componentIndex).setComponentName(componentName);
    if (remoteDebuggerPort != null) {
        instanceInfoBuilder.setRemoteDebuggerPort(remoteDebuggerPort);
    }
    PhysicalPlans.InstanceInfo instanceInfo = instanceInfoBuilder.build();
    PhysicalPlans.Instance instance = PhysicalPlans.Instance.newBuilder().setInstanceId(instanceId).setStmgrId(streamId).setInfo(instanceInfo).build();
    // Init the logging setting and redirect the stdout and stderr to logging
    // For now we just set the logging level as INFO; later we may accept an argument to set it.
    Level loggingLevel = Level.INFO;
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    // Log to file and TManager
    LoggingHelper.loggerInit(loggingLevel, true);
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(instanceId, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
    LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
    String logMsg = "\nStarting instance " + instanceId + " for topology " + topologyName + " and topologyId " + topologyId + " for component " + componentName + " with taskId " + taskId + " and componentIndex " + componentIndex + " and streamManagerId " + streamId + " and streamManagerPort " + streamPort + " and metricsManagerPort " + metricsPort;
    if (remoteDebuggerPort != null) {
        logMsg += " and remoteDebuggerPort " + remoteDebuggerPort;
    }
    LOG.info(logMsg);
    LOG.info("System Config: " + systemConfig.toString());
    HeronInstance heronInstance = new HeronInstance(topologyName, topologyId, instance, streamPort, metricsPort);
    heronInstance.start();
}

Example 8 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class MetricsCacheTest method testMetricCache.

@Test
public void testMetricCache() throws IOException {
    // prepare config files
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(CONFIG_SYSTEM_PATH, true).build();
    MetricsSinksConfig sinksConfig = new MetricsSinksConfig(CONFIG_SINK_PATH, null);
    // initialize metric cache, except looper
    MetricsCache mc = new MetricsCache(systemConfig, sinksConfig, new NIOLooper());
    mc.addMetrics(TopologyManager.PublishMetrics.newBuilder().addMetrics(TopologyManager.MetricDatum.newBuilder().setComponentName("c1").setInstanceId("i1").setName("__jvm-uptime-secs").setTimestamp(System.currentTimeMillis()).setValue("0.1")).addExceptions(TopologyManager.TmanagerExceptionLog.newBuilder().setComponentName("c1").setHostname("h1").setInstanceId("i1").setStacktrace("s1").setLogging("l1").setCount(1).setFirsttime(String.valueOf(System.currentTimeMillis())).setLasttime(String.valueOf(System.currentTimeMillis()))).build());
    // query last 10 seconds
    TopologyManager.MetricResponse response = mc.getMetrics(TopologyManager.MetricRequest.newBuilder().setComponentName("c1").addInstanceId("i1").setInterval(10).addMetric("__jvm-uptime-secs").build());
    assertEquals(response.getMetricCount(), 1);
    assertEquals(response.getMetric(0).getInstanceId(), "i1");
    assertEquals(response.getMetric(0).getMetricCount(), 1);
    assertEquals(response.getMetric(0).getMetric(0).getName(), "__jvm-uptime-secs");
    assertEquals(response.getMetric(0).getMetric(0).getValue(), "0.1");
}

Example 9 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class AbstractNetworkTest method runStreamManagerClient.

StreamManagerClient runStreamManagerClient() {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    HeronSocketOptions socketOptions = new HeronSocketOptions(systemConfig.getInstanceNetworkWriteBatchSize(), systemConfig.getInstanceNetworkWriteBatchTime(), systemConfig.getInstanceNetworkReadBatchSize(), systemConfig.getInstanceNetworkReadBatchTime(), systemConfig.getInstanceNetworkOptionsSocketSendBufferSize(), systemConfig.getInstanceNetworkOptionsSocketReceivedBufferSize(), systemConfig.getInstanceNetworkOptionsMaximumPacketSize());
    final NIOLooper nioLooper = (NIOLooper) communicatorTester.getTestLooper();
    streamManagerClient = new StreamManagerClient(nioLooper, HOST, serverPort, "topology-name", "topologyId", UnitTestHelper.getInstance("bolt-id"), communicatorTester.getInStreamQueue(), communicatorTester.getOutStreamQueue(), communicatorTester.getInControlQueue(), socketOptions, gatewayMetrics);
    Runnable r = new Runnable() {

        @Override
        public void run() {
            try {
                streamManagerClient.start();
                nioLooper.loop();
            } finally {
                streamManagerClient.stop();
                nioLooper.exitLoop();
            }
        }
    };
    threadPool.execute(r);
    return streamManagerClient;
}

Example 10 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class GatewayMetrics method registerMetrics.

/**
 * Register default Gateway Metrics to given MetricsCollector
 *
 * @param metricsCollector the MetricsCollector to register Metrics on
 */
public void registerMetrics(MetricsCollector metricsCollector) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = (int) systemConfig.getHeronMetricsExportInterval().getSeconds();
    metricsCollector.registerMetric("__gateway-received-packets-size", receivedPacketsSize, interval);
    metricsCollector.registerMetric("__gateway-sent-packets-size", sentPacketsSize, interval);
    metricsCollector.registerMetric("__gateway-received-packets-count", receivedPacketsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-packets-count", sentPacketsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-metrics-size", sentMetricsSize, interval);
    metricsCollector.registerMetric("__gateway-sent-metrics-packets-count", sentMetricsPacketsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-metrics-count", sentMetricsCount, interval);
    metricsCollector.registerMetric("__gateway-sent-exceptions-count", sentExceptionsCount, interval);
    metricsCollector.registerMetric("__gateway-in-stream-queue-size", inStreamQueueSize, interval);
    metricsCollector.registerMetric("__gateway-out-stream-queue-size", outStreamQueueSize, interval);
    metricsCollector.registerMetric("__gateway-in-stream-queue-expected-capacity", inStreamQueueExpectedCapacity, interval);
    metricsCollector.registerMetric("__gateway-out-stream-queue-expected-capacity", outStreamQueueExpectedCapacity, interval);
    metricsCollector.registerMetric("__gateway-in-queue-full-count", inQueueFullCount, interval);
}