use of org.apache.heron.common.utils.logging.ErrorReportLoggingHandler in project heron by twitter.
the class MetricsManager method main.
public static void main(String[] args) throws Exception {
final Options options = constructOptions();
final Options helpOptions = constructHelpOptions();
final CommandLineParser parser = new DefaultParser();
// parse the help options first.
CommandLine cmd = parser.parse(helpOptions, args, true);
if (cmd.hasOption("h")) {
usage(options);
return;
}
try {
// Now parse the required options
cmd = parser.parse(options, args);
} catch (ParseException pe) {
usage(options);
throw new RuntimeException("Error parsing command line options: ", pe);
}
String metricsmgrId = cmd.getOptionValue("id");
int metricsPort = Integer.parseInt(cmd.getOptionValue("port"));
String topologyName = cmd.getOptionValue("topology");
String topologyId = cmd.getOptionValue("topology-id");
String systemConfigFilename = cmd.getOptionValue("system-config-file");
String overrideConfigFilename = cmd.getOptionValue("override-config-file");
String metricsSinksConfigFilename = cmd.getOptionValue("sink-config-file");
String cluster = cmd.getOptionValue("cluster");
String role = cmd.getOptionValue("role");
String environment = cmd.getOptionValue("environment");
SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).putAll(overrideConfigFilename, true).build();
// Add the SystemConfig into SingletonRegistry
SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
// Init the logging setting and redirect the stdout and stderr to logging
// For now we just set the logging level as INFO; later we may accept an argument to set it.
Level loggingLevel = Level.INFO;
String loggingDir = systemConfig.getHeronLoggingDirectory();
// Log to file and TManager
LoggingHelper.loggerInit(loggingLevel, true);
LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(metricsmgrId, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
LOG.info(String.format("Starting Metrics Manager for topology %s with topologyId %s with " + "Metrics Manager Id %s, Metrics Manager Port: %d, for cluster/role/env %s.", topologyName, topologyId, metricsmgrId, metricsPort, String.format("%s/%s/%s", cluster, role, environment)));
LOG.info("System Config: " + systemConfig);
// Populate the config
MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename, overrideConfigFilename);
LOG.info("Sinks Config:" + sinksConfig.toString());
MetricsManager metricsManager = new MetricsManager(topologyName, cluster, role, environment, METRICS_MANAGER_HOST, metricsPort, metricsmgrId, systemConfig, sinksConfig);
metricsManager.start();
LOG.info("Loops terminated. Metrics Manager exits.");
}
use of org.apache.heron.common.utils.logging.ErrorReportLoggingHandler in project heron by twitter.
the class HeronInstance method main.
public static void main(String[] args) throws IOException {
CommandLine commandLine = parseCommandLineArgs(args);
String topologyName = commandLine.getOptionValue(CommandLineOptions.TOPOLOGY_NAME_OPTION);
String topologyId = commandLine.getOptionValue(CommandLineOptions.TOPOLOGY_ID_OPTION);
String instanceId = commandLine.getOptionValue(CommandLineOptions.INSTANCE_ID_OPTION);
String componentName = commandLine.getOptionValue(CommandLineOptions.COMPONENT_NAME_OPTION);
Integer taskId = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.TASK_ID_OPTION));
Integer componentIndex = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.COMPONENT_INDEX_OPTION));
String streamId = commandLine.getOptionValue(CommandLineOptions.STMGR_ID_OPTION);
Integer streamPort = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.STMGR_PORT_OPTION));
Integer metricsPort = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.METRICS_MGR_PORT_OPTION));
String systemConfigFile = commandLine.getOptionValue(CommandLineOptions.SYSTEM_CONFIG_FILE);
String overrideConfigFile = commandLine.getOptionValue(CommandLineOptions.OVERRIDE_CONFIG_FILE);
Integer remoteDebuggerPort = null;
if (commandLine.hasOption(CommandLineOptions.REMOTE_DEBUGGER_PORT)) {
remoteDebuggerPort = Integer.parseInt(commandLine.getOptionValue(CommandLineOptions.REMOTE_DEBUGGER_PORT));
}
SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFile, true).putAll(overrideConfigFile, true).build();
// Add the SystemConfig into SingletonRegistry
SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
// Create the protobuf Instance
PhysicalPlans.InstanceInfo.Builder instanceInfoBuilder = PhysicalPlans.InstanceInfo.newBuilder().setTaskId(taskId).setComponentIndex(componentIndex).setComponentName(componentName);
if (remoteDebuggerPort != null) {
instanceInfoBuilder.setRemoteDebuggerPort(remoteDebuggerPort);
}
PhysicalPlans.InstanceInfo instanceInfo = instanceInfoBuilder.build();
PhysicalPlans.Instance instance = PhysicalPlans.Instance.newBuilder().setInstanceId(instanceId).setStmgrId(streamId).setInfo(instanceInfo).build();
// Init the logging setting and redirect the stdout and stderr to logging
// For now we just set the logging level as INFO; later we may accept an argument to set it.
Level loggingLevel = Level.INFO;
String loggingDir = systemConfig.getHeronLoggingDirectory();
// Log to file and TManager
LoggingHelper.loggerInit(loggingLevel, true);
LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(instanceId, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
String logMsg = "\nStarting instance " + instanceId + " for topology " + topologyName + " and topologyId " + topologyId + " for component " + componentName + " with taskId " + taskId + " and componentIndex " + componentIndex + " and streamManagerId " + streamId + " and streamManagerPort " + streamPort + " and metricsManagerPort " + metricsPort;
if (remoteDebuggerPort != null) {
logMsg += " and remoteDebuggerPort " + remoteDebuggerPort;
}
LOG.info(logMsg);
LOG.info("System Config: " + systemConfig.toString());
HeronInstance heronInstance = new HeronInstance(topologyName, topologyId, instance, streamPort, metricsPort);
heronInstance.start();
}
use of org.apache.heron.common.utils.logging.ErrorReportLoggingHandler in project heron by twitter.
the class MetricsCacheManager method main.
public static void main(String[] args) throws Exception {
Options options = constructOptions();
Options helpOptions = constructHelpOptions();
CommandLineParser parser = new DefaultParser();
// parse the help options first.
CommandLine cmd = parser.parse(helpOptions, args, true);
if (cmd.hasOption("h")) {
usage(options);
return;
}
try {
// Now parse the required options
cmd = parser.parse(options, args);
} catch (ParseException e) {
usage(options);
throw new RuntimeException("Error parsing command line options: ", e);
}
Level logLevel = Level.INFO;
if (cmd.hasOption("v")) {
logLevel = Level.ALL;
}
String cluster = cmd.getOptionValue("cluster");
String role = cmd.getOptionValue("role");
String environ = cmd.getOptionValue("environment");
int serverPort = Integer.valueOf(cmd.getOptionValue("server_port"));
int statsPort = Integer.valueOf(cmd.getOptionValue("stats_port"));
String systemConfigFilename = cmd.getOptionValue("system_config_file");
String overrideConfigFilename = cmd.getOptionValue("override_config_file");
String metricsSinksConfigFilename = cmd.getOptionValue("sink_config_file");
String topologyName = cmd.getOptionValue("topology_name");
String topologyId = cmd.getOptionValue("topology_id");
String metricsCacheMgrId = cmd.getOptionValue("metricscache_id");
// read heron internal config file
SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(systemConfigFilename, true).putAll(overrideConfigFilename, true).build();
// Log to file and sink(exception)
LoggingHelper.loggerInit(logLevel, true);
LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(metricsCacheMgrId, systemConfig.getHeronLoggingDirectory(), true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
LOG.info(String.format("Starting MetricsCache for topology %s with topologyId %s with " + "MetricsCache Id %s, server port: %d.", topologyName, topologyId, metricsCacheMgrId, serverPort));
LOG.info("System Config: " + systemConfig);
// read sink config file
MetricsSinksConfig sinksConfig = new MetricsSinksConfig(metricsSinksConfigFilename, overrideConfigFilename);
LOG.info("Sinks Config: " + sinksConfig.toString());
// build config from cli
Config config = Config.toClusterMode(Config.newBuilder().putAll(ConfigLoader.loadClusterConfig()).putAll(Config.newBuilder().put(Key.CLUSTER, cluster).put(Key.ROLE, role).put(Key.ENVIRON, environ).build()).putAll(Config.newBuilder().put(Key.TOPOLOGY_NAME, topologyName).put(Key.TOPOLOGY_ID, topologyId).build()).build());
LOG.info("Cli Config: " + config.toString());
// build metricsCache location
TopologyManager.MetricsCacheLocation metricsCacheLocation = TopologyManager.MetricsCacheLocation.newBuilder().setTopologyName(topologyName).setTopologyId(topologyId).setHost(InetAddress.getLocalHost().getHostName()).setControllerPort(// not used for metricscache
-1).setServerPort(serverPort).setStatsPort(statsPort).build();
MetricsCacheManager metricsCacheManager = new MetricsCacheManager(topologyName, METRICS_CACHE_HOST, serverPort, statsPort, systemConfig, sinksConfig, config, metricsCacheLocation);
metricsCacheManager.start();
LOG.info("Loops terminated. MetricsCache Manager exits.");
}
use of org.apache.heron.common.utils.logging.ErrorReportLoggingHandler in project heron by twitter.
the class CheckpointManager method main.
public static void main(String[] args) throws IOException, ParseException, CheckpointManagerException {
Options options = constructOptions();
Options helpOptions = constructHelpOptions();
CommandLineParser parser = new DefaultParser();
// parse the help options first.
CommandLine cmd = parser.parse(helpOptions, args, true);
if (cmd.hasOption("h")) {
usage(options);
return;
}
try {
// Now parse the required options
cmd = parser.parse(options, args);
} catch (ParseException e) {
usage(options);
throw new RuntimeException("Error parsing command line options ", e);
}
String topologyName = cmd.getOptionValue("topologyname");
String topologyId = cmd.getOptionValue("topologyid");
String ckptmgrId = cmd.getOptionValue("ckptmgrid");
int port = Integer.parseInt(cmd.getOptionValue("ckptmgrport"));
String stateConfigFilename = cmd.getOptionValue("ckptmgrconfig");
String overriddenConfigFilename = cmd.getOptionValue("ckptMgrOverridenConfig");
String heronInternalConfig = cmd.getOptionValue("heroninternalconfig");
SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(heronInternalConfig, true).build();
CheckpointManagerConfig ckptmgrConfig = CheckpointManagerConfig.newBuilder(true).putAll(stateConfigFilename, true).override(overriddenConfigFilename).build();
// Add the SystemConfig into SingletonRegistry
SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
// Init the logging setting and redirect the stdout and stderr to logging
// For now we just set the logging level as INFO; later we may accept an argument to set it.
Level loggingLevel = Level.INFO;
String loggingDir = systemConfig.getHeronLoggingDirectory();
// Log to file and TManager
LoggingHelper.loggerInit(loggingLevel, true);
LoggingHelper.addLoggingHandler(LoggingHelper.getFileHandler(ckptmgrId, loggingDir, true, systemConfig.getHeronLoggingMaximumSize(), systemConfig.getHeronLoggingMaximumFiles()));
LoggingHelper.addLoggingHandler(new ErrorReportLoggingHandler());
// Start the actual things
LOG.info(String.format("Starting topology %s with topologyId %s with " + "Checkpoint Manager Id %s, Port: %d.", topologyName, topologyId, ckptmgrId, port));
LOG.info("System Config: " + systemConfig);
LOG.info(() -> "Checkpoint Manager Config: " + ckptmgrConfig);
CheckpointManager checkpointManager = new CheckpointManager();
checkpointManager.init(topologyName, topologyId, ckptmgrId, CHECKPOINT_MANAGER_HOST, port, systemConfig, ckptmgrConfig);
checkpointManager.startAndLoop();
LOG.info("Loops terminated. Exiting.");
}
Aggregations