Examples with SystemConfig com.twitter.heron.common.config.SystemConfig used on opensource projects

Example 11 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class FullSpoutMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = systemConfig.getHeronMetricsExportIntervalSec();
    topologyContext.registerMetric("__ack-count", ackCount, interval);
    topologyContext.registerMetric("__complete-latency", completeLatency, interval);
    topologyContext.registerMetric("__fail-latency", failLatency, interval);
    topologyContext.registerMetric("__fail-count", failCount, interval);
    topologyContext.registerMetric("__timeout-count", timeoutCount, interval);
    topologyContext.registerMetric("__emit-count", emitCount, interval);
    topologyContext.registerMetric("__next-tuple-latency", nextTupleLatency, interval);
    topologyContext.registerMetric("__next-tuple-count", nextTupleCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
    topologyContext.registerMetric("__pending-acked-count", pendingTuplesCount, interval);
    topologyContext.registerMetric("__tuple-serialization-time-ns", serializationTimeNs, interval);
}

Example 12 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class ConnectTest method runStreamManagerClient.

void runStreamManagerClient() {
    Runnable r = new Runnable() {

        @Override
        public void run() {
            try {
                SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
                HeronSocketOptions socketOptions = new HeronSocketOptions(systemConfig.getInstanceNetworkWriteBatchSizeBytes(), systemConfig.getInstanceNetworkWriteBatchTimeMs(), systemConfig.getInstanceNetworkReadBatchSizeBytes(), systemConfig.getInstanceNetworkReadBatchTimeMs(), systemConfig.getInstanceNetworkOptionsSocketSendBufferSizeBytes(), systemConfig.getInstanceNetworkOptionsSocketReceivedBufferSizeBytes());
                streamManagerClient = new StreamManagerClient(nioLooper, HOST, serverPort, "topology-name", "topologyId", UnitTestHelper.getInstance("bolt-id"), inStreamQueue, outStreamQueue, inControlQueue, socketOptions, gatewayMetrics);
                streamManagerClient.start();
                nioLooper.loop();
            } finally {
                streamManagerClient.stop();
                nioLooper.exitLoop();
            }
        }
    };
    threadsPool.execute(r);
}

Example 13 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class MetricsCacheTest method testMetricCache.

@Test
public void testMetricCache() throws IOException {
    // prepare config files
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(CONFIG_SYSTEM_PATH, true).build();
    MetricsSinksConfig sinksConfig = new MetricsSinksConfig(CONFIG_SINK_PATH);
    // initialize metric cache, except looper
    MetricsCache mc = new MetricsCache(systemConfig, sinksConfig, new NIOLooper());
    mc.addMetrics(TopologyMaster.PublishMetrics.newBuilder().addMetrics(TopologyMaster.MetricDatum.newBuilder().setComponentName("c1").setInstanceId("i1").setName("__jvm-uptime-secs").setTimestamp(System.currentTimeMillis()).setValue("0.1")).addExceptions(TopologyMaster.TmasterExceptionLog.newBuilder().setComponentName("c1").setHostname("h1").setInstanceId("i1").setStacktrace("s1").setLogging("l1").setCount(1).setFirsttime(String.valueOf(System.currentTimeMillis())).setLasttime(String.valueOf(System.currentTimeMillis()))).build());
    // query last 10 seconds
    TopologyMaster.MetricResponse response = mc.getMetrics(TopologyMaster.MetricRequest.newBuilder().setComponentName("c1").addInstanceId("i1").setInterval(10).addMetric("__jvm-uptime-secs").build());
    assertEquals(response.getMetricCount(), 1);
    assertEquals(response.getMetric(0).getInstanceId(), "i1");
    assertEquals(response.getMetric(0).getMetricCount(), 1);
    assertEquals(response.getMetric(0).getMetric(0).getName(), "__jvm-uptime-secs");
    assertEquals(response.getMetric(0).getMetric(0).getValue(), "0.1");
}

Example 14 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class SchedulerMain method setupLogging.

// Set up logging based on the Config
private static void setupLogging(Config config) throws IOException {
    String systemConfigFilename = Context.systemConfigFile(config);
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).build();
    // Init the logging setting and redirect the stdout and stderr to logging
    // For now we just set the logging level as INFO; later we may accept an argument to set it.
    Level loggingLevel = Level.INFO;
    if (Context.verbose(config).booleanValue()) {
        loggingLevel = Level.FINE;
    }
    // TODO(mfu): The folder creation may be duplicated with heron-executor in future
    // TODO(mfu): Remove the creation in future if feasible
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    if (!FileUtils.isDirectoryExists(loggingDir)) {
        FileUtils.createDirectory(loggingDir);
    }
    // Log to file
    LoggingHelper.loggerInit(loggingLevel, true);
    // TODO(mfu): Pass the scheduler id from cmd
    String processId = String.format("%s-%s-%s", "heron", Context.topologyName(config), "scheduler");
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(processId, loggingDir, true, systemConfig.getHeronLoggingMaximumSizeMb() * Constants.MB_TO_BYTES, systemConfig.getHeronLoggingMaximumFiles()));
    LOG.info("Logging setup done.");
}