Examples with SystemConfig org.apache.heron.common.config.SystemConfig used on opensource projects

Example 11 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class HealthManager method main.

public static void main(String[] args) throws Exception {
    CommandLineParser parser = new DefaultParser();
    Options slaManagerCliOptions = constructCliOptions();
    // parse the help options first.
    Options helpOptions = constructHelpOptions();
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(slaManagerCliOptions);
        return;
    }
    try {
        cmd = parser.parse(slaManagerCliOptions, args);
    } catch (ParseException e) {
        usage(slaManagerCliOptions);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
    HealthManagerMode mode = HealthManagerMode.cluster;
    if (hasOption(cmd, CliArgs.MODE)) {
        mode = HealthManagerMode.valueOf(getOptionValue(cmd, CliArgs.MODE));
    }
    Config config;
    switch(mode) {
        case cluster:
            config = Config.toClusterMode(Config.newBuilder().putAll(ConfigLoader.loadClusterConfig()).putAll(commandLineConfigs(cmd)).build());
            break;
        case local:
            if (!hasOption(cmd, CliArgs.HERON_HOME) || !hasOption(cmd, CliArgs.CONFIG_PATH)) {
                throw new IllegalArgumentException("Missing heron_home or config_path argument");
            }
            String heronHome = getOptionValue(cmd, CliArgs.HERON_HOME);
            String configPath = getOptionValue(cmd, CliArgs.CONFIG_PATH);
            config = Config.toLocalMode(Config.newBuilder().putAll(ConfigLoader.loadConfig(heronHome, configPath, null, null)).putAll(commandLineConfigs(cmd)).build());
            break;
        default:
            throw new IllegalArgumentException("Invalid mode: " + getOptionValue(cmd, CliArgs.MODE));
    }
    setupLogging(cmd, config);
    LOG.fine(Arrays.toString(cmd.getOptions()));
    // Add the SystemConfig into SingletonRegistry
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(Context.systemFile(config), true).putAll(Context.overrideFile(config), true).build();
    SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
    LOG.info("Static Heron config loaded successfully ");
    LOG.fine(config.toString());
    // load the default config value and override with any command line values
    String metricSourceClassName = config.getStringValue(PolicyConfigKey.METRIC_SOURCE_TYPE.key());
    metricSourceClassName = getOptionValue(cmd, CliArgs.METRIC_SOURCE_TYPE, metricSourceClassName);
    String metricsUrl = config.getStringValue(PolicyConfigKey.METRIC_SOURCE_URL.key());
    metricsUrl = getOptionValue(cmd, CliArgs.METRIC_SOURCE_URL, metricsUrl);
    // metrics reporting thread
    HealthManagerMetrics publishingMetrics = new HealthManagerMetrics(Integer.valueOf(getOptionValue(cmd, CliArgs.METRICSMGR_PORT)));
    AbstractModule module = buildBaseModule(metricsUrl, metricSourceClassName, publishingMetrics);
    HealthManager healthManager = new HealthManager(config, module);
    LOG.info("Initializing health manager");
    healthManager.initialize();
    LOG.info("Starting Health Manager");
    PoliciesExecutor policyExecutor = new PoliciesExecutor(healthManager.healthPolicies);
    ScheduledFuture<?> future = policyExecutor.start();
    LOG.info("Starting Health Manager metric posting thread");
    new Thread(publishingMetrics).start();
    try {
        future.get();
    } finally {
        policyExecutor.destroy();
        publishingMetrics.close();
    }
}

Example 12 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class HealthManager method setupLogging.

private static void setupLogging(CommandLine cmd, Config config) throws IOException {
    String systemConfigFilename = Context.systemConfigFile(config);
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).build();
    Boolean verbose = hasOption(cmd, CliArgs.VERBOSE);
    Level loggingLevel = Level.INFO;
    if (verbose) {
        loggingLevel = Level.FINE;
    }
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    LoggingHelper.loggerInit(loggingLevel, true);
    String fileName = String.format("%s-%s-%s", "heron", Context.topologyName(config), "healthmgr");
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(fileName, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
    LOG.info("Logging setup done.");
}

Example 13 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class FullSpoutMetrics method registerMetrics.

public void registerMetrics(TopologyContextImpl topologyContext) {
    SystemConfig systemConfig = (SystemConfig) SingletonRegistry.INSTANCE.getSingleton(SystemConfig.HERON_SYSTEM_CONFIG);
    int interval = (int) systemConfig.getHeronMetricsExportInterval().getSeconds();
    topologyContext.registerMetric("__ack-count", ackCount, interval);
    topologyContext.registerMetric("__complete-latency", completeLatency, interval);
    topologyContext.registerMetric("__fail-latency", failLatency, interval);
    topologyContext.registerMetric("__fail-count", failCount, interval);
    topologyContext.registerMetric("__timeout-count", timeoutCount, interval);
    topologyContext.registerMetric("__emit-count", emitCount, interval);
    topologyContext.registerMetric("__next-tuple-latency", nextTupleLatency, interval);
    topologyContext.registerMetric("__next-tuple-count", nextTupleCount, interval);
    topologyContext.registerMetric("__out-queue-full-count", outQueueFullCount, interval);
    topologyContext.registerMetric("__pending-acked-count", pendingTuplesCount, interval);
    topologyContext.registerMetric("__tuple-serialization-time-ns", serializationTimeNs, interval);
    topologyContext.registerMetric("__task-run-count", taskRunCount, interval);
    topologyContext.registerMetric("__produce-tuple-count", produceTupleCount, interval);
    topologyContext.registerMetric("__continue-work-count", continueWorkCount, interval);
    // The following metrics measure the rate at which tuples are added to the outgoing
    // queues at spouts and the sizes of these queues. This allows us to measure whether
    // the gateway thread pulls out tuples from the queue fast enough, thereby preventing
    // the spout from becoming a bottleneck.
    topologyContext.registerMetric("__data-tuple-added-to-outgoing-queue/default", tupleAddedToQueue, interval);
    topologyContext.registerMetric("__average-tuple-size-added-queue/default", tupleSize, interval);
}

Example 14 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class MetricsCacheManager method main.

public static void main(String[] args) throws Exception {
    Options options = constructOptions();
    Options helpOptions = constructHelpOptions();
    CommandLineParser parser = new DefaultParser();
    // parse the help options first.
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(options);
        return;
    }
    try {
        // Now parse the required options
        cmd = parser.parse(options, args);
    } catch (ParseException e) {
        usage(options);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
    Level logLevel = Level.INFO;
    if (cmd.hasOption("v")) {
        logLevel = Level.ALL;
    }
    String cluster = cmd.getOptionValue("cluster");
    String role = cmd.getOptionValue("role");
    String environ = cmd.getOptionValue("environment");
    int serverPort = Integer.valueOf(cmd.getOptionValue("server_port"));
    int statsPort = Integer.valueOf(cmd.getOptionValue("stats_port"));
    String systemConfigFilename = cmd.getOptionValue("system_config_file");
    String overrideConfigFilename = cmd.getOptionValue("override_config_file");
    String metricsSinksConfigFilename = cmd.getOptionValue("sink_config_file");
    String topologyName = cmd.getOptionValue("topology_name");
    String topologyId = cmd.getOptionValue("topology_id");
    String metricsCacheMgrId = cmd.getOptionValue("metricscache_id");
    // read heron internal config file
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).putAll(overrideConfigFilename, true).build();
    // Log to file and sink(exception)
    LoggingHelper.loggerInit(logLevel, true);
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(metricsCacheMgrId, systemConfig.getHeronLoggingDirectory(), true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
    LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
    LOG.info(String.format("Starting MetricsCache for topology %s with topologyId %s with " + "MetricsCache Id %s, server port: %d.", topologyName, topologyId, metricsCacheMgrId, serverPort));
    LOG.info("System Config: " + systemConfig);
    // read sink config file
    MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename, overrideConfigFilename);
    LOG.info("Sinks Config: " + sinksConfig.toString());
    // build config from cli
    Config config = Config.toClusterMode(Config.newBuilder().putAll(ConfigLoader.loadClusterConfig()).putAll(Config.newBuilder().put(Key.CLUSTER, cluster).put(Key.ROLE, role).put(Key.ENVIRON, environ).build()).putAll(Config.newBuilder().put(Key.TOPOLOGY_NAME, topologyName).put(Key.TOPOLOGY_ID, topologyId).build()).build());
    LOG.info("Cli Config: " + config.toString());
    // build metricsCache location
    TopologyManager.MetricsCacheLocation metricsCacheLocation = TopologyManager.MetricsCacheLocation.newBuilder().setTopologyName(topologyName).setTopologyId(topologyId).setHost(InetAddress.getLocalHost().getHostName()).setControllerPort(// not used for metricscache
    -1).setServerPort(serverPort).setStatsPort(statsPort).build();
    MetricsCacheManager metricsCacheManager = new MetricsCacheManager(topologyName, METRICS_CACHE_HOST, serverPort, statsPort, systemConfig, sinksConfig, config, metricsCacheLocation);
    metricsCacheManager.start();
    LOG.info("Loops terminated. MetricsCache Manager exits.");
}

Example 15 with SystemConfig

use of org.apache.heron.common.config.SystemConfig in project heron by twitter.

the class CheckpointManager method main.

public static void main(String[] args) throws IOException, ParseException, CheckpointManagerException {
    Options options = constructOptions();
    Options helpOptions = constructHelpOptions();
    CommandLineParser parser = new DefaultParser();
    // parse the help options first.
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(options);
        return;
    }
    try {
        // Now parse the required options
        cmd = parser.parse(options, args);
    } catch (ParseException e) {
        usage(options);
        throw new RuntimeException("Error parsing command line options ", e);
    }
    String topologyName = cmd.getOptionValue("topologyname");
    String topologyId = cmd.getOptionValue("topologyid");
    String ckptmgrId = cmd.getOptionValue("ckptmgrid");
    int port = Integer.parseInt(cmd.getOptionValue("ckptmgrport"));
    String stateConfigFilename = cmd.getOptionValue("ckptmgrconfig");
    String overriddenConfigFilename = cmd.getOptionValue("ckptMgrOverridenConfig");
    String heronInternalConfig = cmd.getOptionValue("heroninternalconfig");
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(heronInternalConfig, true).build();
    CheckpointManagerConfig ckptmgrConfig = CheckpointManagerConfig.newBuilder(true).putAll(stateConfigFilename, true).override(overriddenConfigFilename).build();
    // Add the SystemConfig into SingletonRegistry
    SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
    // Init the logging setting and redirect the stdout and stderr to logging
    // For now we just set the logging level as INFO; later we may accept an argument to set it.
    Level loggingLevel = Level.INFO;
    String loggingDir = systemConfig.getHeronLoggingDirectory();
    // Log to file and TManager
    LoggingHelper.loggerInit(loggingLevel, true);
    LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(ckptmgrId, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
    LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
    // Start the actual things
    LOG.info(String.format("Starting topology %s with topologyId %s with " + "Checkpoint Manager Id %s, Port: %d.", topologyName, topologyId, ckptmgrId, port));
    LOG.info("System Config: " + systemConfig);
    LOG.info(() -> "Checkpoint Manager Config: " + ckptmgrConfig);
    CheckpointManager checkpointManager = new CheckpointManager();
    checkpointManager.init(topologyName, topologyId, ckptmgrId, CHECKPOINT_MANAGER_HOST, port, systemConfig, ckptmgrConfig);
    checkpointManager.startAndLoop();
    LOG.info("Loops terminated. Exiting.");
}