Examples with SystemConfig com.twitter.heron.common.config.SystemConfig used on opensource projects

Example 6 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class HandleWriteTest method runStreamManagerClient.

void runStreamManagerClient() {
    Runnable r = new Runnable() {

        @Override
        public void run() {
            try {
                SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
                HeronSocketOptions socketOptions = new HeronSocketOptions(systemConfig.getInstanceNetworkWriteBatchSizeBytes(), systemConfig.getInstanceNetworkWriteBatchTimeMs(), systemConfig.getInstanceNetworkReadBatchSizeBytes(), systemConfig.getInstanceNetworkReadBatchTimeMs(), systemConfig.getInstanceNetworkOptionsSocketSendBufferSizeBytes(), systemConfig.getInstanceNetworkOptionsSocketReceivedBufferSizeBytes());
                streamManagerClient = new StreamManagerClient(nioLooper, HOST, serverPort, "topology-name", "topologyId", UnitTestHelper.getInstance("bolt-id"), inStreamQueue, outStreamQueue, inControlQueue, socketOptions, gatewayMetrics);
                streamManagerClient.start();
                nioLooper.loop();
            } finally {
                streamManagerClient.stop();
                nioLooper.exitLoop();
            }
        }
    };
    threadsPool.execute(r);
}

Example 7 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class HandleReadTest method runStreamManagerClient.

void runStreamManagerClient() {
    Runnable r = new Runnable() {

        @Override
        public void run() {
            try {
                SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
                HeronSocketOptions socketOptions = new HeronSocketOptions(systemConfig.getInstanceNetworkWriteBatchSizeBytes(), systemConfig.getInstanceNetworkWriteBatchTimeMs(), systemConfig.getInstanceNetworkReadBatchSizeBytes(), systemConfig.getInstanceNetworkReadBatchTimeMs(), systemConfig.getInstanceNetworkOptionsSocketSendBufferSizeBytes(), systemConfig.getInstanceNetworkOptionsSocketReceivedBufferSizeBytes());
                streamManagerClient = new StreamManagerClient(nioLooper, HOST, serverPort, "topology-name", "topologyId", UnitTestHelper.getInstance("bolt-id"), inStreamQueue, outStreamQueue, inControlQueue, socketOptions, gatewayMetrics);
                streamManagerClient.start();
                nioLooper.loop();
            } finally {
                streamManagerClient.stop();
                nioLooper.exitLoop();
            }
        }
    };
    threadsPool.execute(r);
}

Example 8 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class MetricsCacheManager method main.

public static void main(String[] args) throws Exception {
    Options options = constructOptions();
    Options helpOptions = constructHelpOptions();
    CommandLineParser parser = new DefaultParser();
    // parse the help options first.
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(options);
        return;
    }
    try {
        // Now parse the required options
        cmd = parser.parse(options, args);
    } catch (ParseException e) {
        usage(options);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
    Level logLevel = Level.INFO;
    if (cmd.hasOption("v")) {
        logLevel = Level.ALL;
    }
    String cluster = cmd.getOptionValue("cluster");
    String role = cmd.getOptionValue("role");
    String environ = cmd.getOptionValue("environment");
    int masterPort = Integer.valueOf(cmd.getOptionValue("master_port"));
    int statsPort = Integer.valueOf(cmd.getOptionValue("stats_port"));
    String systemConfigFilename = cmd.getOptionValue("system_config_file");
    String metricsSinksConfigFilename = cmd.getOptionValue("sink_config_file");
    String topologyName = cmd.getOptionValue("topology_name");
    String topologyId = cmd.getOptionValue("topology_id");
    String metricsCacheMgrId = cmd.getOptionValue("metricscache_id");
    // read heron internal config file
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).build();
    // Log to file and sink(exception)
    LoggingHelper.loggerInit(logLevel, true);
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(metricsCacheMgrId, systemConfig.getHeronLoggingDirectory(), true, systemConfig.getHeronLoggingMaximumSizeMb() * Constants.MB_TO_BYTES, systemConfig.getHeronLoggingMaximumFiles()));
    LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
    LOG.info(String.format("Starting MetricsCache for topology %s with topologyId %s with " + "MetricsCache Id %s, master port: %d.", topologyName, topologyId, metricsCacheMgrId, masterPort));
    LOG.info("System Config: " + systemConfig);
    // read sink config file
    MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename);
    LOG.info("Sinks Config: " + sinksConfig.toString());
    // build config from cli
    Config config = Config.toClusterMode(Config.newBuilder().putAll(ConfigLoader.loadClusterConfig()).putAll(Config.newBuilder().put(Key.CLUSTER, cluster).put(Key.ROLE, role).put(Key.ENVIRON, environ).build()).putAll(Config.newBuilder().put(Key.TOPOLOGY_NAME, topologyName).put(Key.TOPOLOGY_ID, topologyId).build()).build());
    LOG.info("Cli Config: " + config.toString());
    // build metricsCache location
    TopologyMaster.MetricsCacheLocation metricsCacheLocation = TopologyMaster.MetricsCacheLocation.newBuilder().setTopologyName(topologyName).setTopologyId(topologyId).setHost(InetAddress.getLocalHost().getHostName()).setControllerPort(// not used for metricscache
    -1).setMasterPort(masterPort).setStatsPort(statsPort).build();
    MetricsCacheManager metricsCacheManager = new MetricsCacheManager(topologyName, METRICS_CACHE_HOST, masterPort, statsPort, systemConfig, sinksConfig, config, metricsCacheLocation);
    metricsCacheManager.start();
    LOG.info("Loops terminated. MetricsCache Manager exits.");
}

Example 9 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class BoltMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = systemConfig.getHeronMetricsExportIntervalSec();
    topologyContext.registerMetric("__ack-count/default", ackCount, interval);
    topologyContext.registerMetric("__process-latency/default", processLatency, interval);
    topologyContext.registerMetric("__fail-latency/default", failLatency, interval);
    topologyContext.registerMetric("__fail-count/default", failCount, interval);
    topologyContext.registerMetric("__execute-count/default", executeCount, interval);
    topologyContext.registerMetric("__execute-latency/default", executeLatency, interval);
    topologyContext.registerMetric("__emit-count/default", emitCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
}

Example 10 with SystemConfig

use of com.twitter.heron.common.config.SystemConfig in project heron by twitter.

the class FullBoltMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = systemConfig.getHeronMetricsExportIntervalSec();
    topologyContext.registerMetric("__ack-count", ackCount, interval);
    topologyContext.registerMetric("__process-latency", processLatency, interval);
    topologyContext.registerMetric("__fail-latency", failLatency, interval);
    topologyContext.registerMetric("__fail-count", failCount, interval);
    topologyContext.registerMetric("__execute-count", executeCount, interval);
    topologyContext.registerMetric("__execute-latency", executeLatency, interval);
    topologyContext.registerMetric("__execute-time-ns", executeTimeNs, interval);
    topologyContext.registerMetric("__emit-count", emitCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
    topologyContext.registerMetric("__tuple-deserialization-time-ns", deserializationTimeNs, interval);
    topologyContext.registerMetric("__tuple-serialization-time-ns", serializationTimeNs, interval);
}