Examples with Config com.twitter.heron.spi.common.Config used on opensource projects

Example 56 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class RuntimeManagerMain method main.

public static void main(String[] args) throws ClassNotFoundException, IllegalAccessException, InstantiationException, IOException, ParseException {
    Options options = constructOptions();
    Options helpOptions = constructHelpOptions();
    CommandLineParser parser = new DefaultParser();
    // parse the help options first.
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(options);
        return;
    }
    try {
        // Now parse the required options
        cmd = parser.parse(options, args);
    } catch (ParseException e) {
        usage(options);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
    Boolean verbose = false;
    Level logLevel = Level.INFO;
    if (cmd.hasOption("v")) {
        logLevel = Level.ALL;
        verbose = true;
    }
    // init log
    LoggingHelper.loggerInit(logLevel, false);
    String cluster = cmd.getOptionValue("cluster");
    String role = cmd.getOptionValue("role");
    String environ = cmd.getOptionValue("environment");
    String submitUser = cmd.getOptionValue("submit_user");
    String heronHome = cmd.getOptionValue("heron_home");
    String configPath = cmd.getOptionValue("config_path");
    String overrideConfigFile = cmd.getOptionValue("override_config_file");
    String releaseFile = cmd.getOptionValue("release_file");
    String topologyName = cmd.getOptionValue("topology_name");
    String commandOption = cmd.getOptionValue("command");
    // Optional argument in the case of restart
    // TODO(karthik): convert into CLI
    String containerId = Integer.toString(-1);
    if (cmd.hasOption("container_id")) {
        containerId = cmd.getOptionValue("container_id");
    }
    Boolean dryRun = false;
    if (cmd.hasOption("u")) {
        dryRun = true;
    }
    // Default dry-run output format type
    DryRunFormatType dryRunFormat = DryRunFormatType.TABLE;
    if (dryRun && cmd.hasOption("t")) {
        String format = cmd.getOptionValue("dry_run_format");
        dryRunFormat = DryRunFormatType.getDryRunFormatType(format);
        LOG.fine(String.format("Running dry-run mode using format %s", format));
    }
    Command command = Command.makeCommand(commandOption);
    // add config parameters from the command line
    Config.Builder commandLineConfig = Config.newBuilder().put(Key.CLUSTER, cluster).put(Key.ROLE, role).put(Key.ENVIRON, environ).put(Key.SUBMIT_USER, submitUser).put(Key.DRY_RUN, dryRun).put(Key.DRY_RUN_FORMAT_TYPE, dryRunFormat).put(Key.VERBOSE, verbose).put(Key.TOPOLOGY_CONTAINER_ID, containerId);
    // This is a command line option, but not a valid config key. Hence we don't use Keys
    translateCommandLineConfig(cmd, commandLineConfig);
    Config.Builder topologyConfig = Config.newBuilder().put(Key.TOPOLOGY_NAME, topologyName);
    // build the final config by expanding all the variables
    Config config = Config.toLocalMode(Config.newBuilder().putAll(ConfigLoader.loadConfig(heronHome, configPath, releaseFile, overrideConfigFile)).putAll(commandLineConfig.build()).putAll(topologyConfig.build()).build());
    LOG.fine("Static config loaded successfully ");
    LOG.fine(config.toString());
    /* Meaning of exit status code:
      - status code = 0:
        program exits without error
      - 0 < status code < 100:
        program fails to execute before program execution. For example,
        JVM cannot find or load main class
      - 100 <= status code < 200:
        program fails to launch after program execution. For example,
        topology definition file fails to be loaded
      - status code == 200
        program sends out dry-run response */
    /* Since only stderr is used (by logging), we use stdout here to
       propagate any message back to Python's executor.py (invoke site). */
    // Create a new instance of RuntimeManagerMain
    RuntimeManagerMain runtimeManagerMain = new RuntimeManagerMain(config, command);
    try {
        runtimeManagerMain.manageTopology();
    // SUPPRESS CHECKSTYLE IllegalCatch
    } catch (UpdateDryRunResponse response) {
        LOG.log(Level.FINE, "Sending out dry-run response");
        // Output may contain UTF-8 characters, so we should print using UTF-8 encoding
        PrintStream out = new PrintStream(System.out, true, StandardCharsets.UTF_8.name());
        out.print(DryRunRenders.render(response, Context.dryRunFormatType(config)));
        // SUPPRESS CHECKSTYLE RegexpSinglelineJava
        // Exit with status code 200 to indicate dry-run response is sent out
        System.exit(200);
    // SUPPRESS CHECKSTYLE IllegalCatch
    } catch (Exception e) {
        LOG.log(Level.FINE, "Exception when submitting topology", e);
        System.out.println(e.getMessage());
        // Exit with status code 100 to indicate that error has happened on user-land
        // SUPPRESS CHECKSTYLE RegexpSinglelineJava
        System.exit(100);
    }
}

Example 57 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class LaunchRunnerTest method createRunnerConfig.

private static Config createRunnerConfig() {
    Config config = mock(Config.class);
    when(config.getStringValue(Key.TOPOLOGY_NAME)).thenReturn(TOPOLOGY_NAME);
    when(config.getStringValue(Key.CLUSTER)).thenReturn(CLUSTER);
    when(config.getStringValue(Key.ROLE)).thenReturn(ROLE);
    when(config.getStringValue(Key.ENVIRON)).thenReturn(ENVIRON);
    when(config.getStringValue(Key.SUBMIT_USER)).thenReturn(SUBMIT_USER);
    when(config.getStringValue(Key.BUILD_VERSION)).thenReturn(BUILD_VERSION);
    when(config.getStringValue(Key.BUILD_USER)).thenReturn(BUILD_USER);
    return config;
}

Example 58 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class LaunchRunnerTest method createRunnerRuntime.

private static Config createRunnerRuntime(com.twitter.heron.api.Config topologyConfig) throws Exception {
    Config runtime = spy(Config.newBuilder().build());
    ILauncher launcher = mock(ILauncher.class);
    IPacking packing = mock(IPacking.class);
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    TopologyAPI.Topology topology = createTopology(topologyConfig);
    doReturn(launcher).when(runtime).get(Key.LAUNCHER_CLASS_INSTANCE);
    doReturn(adaptor).when(runtime).get(Key.SCHEDULER_STATE_MANAGER_ADAPTOR);
    doReturn(topology).when(runtime).get(Key.TOPOLOGY_DEFINITION);
    PackingPlan packingPlan = mock(PackingPlan.class);
    when(packingPlan.getContainers()).thenReturn(new HashSet<ContainerPlan>());
    when(packingPlan.getComponentRamDistribution()).thenReturn("ramdist");
    when(packingPlan.getId()).thenReturn("packing_plan_id");
    Set<ContainerPlan> containerPlans = new HashSet<>();
    // just need it to be of size 1
    containerPlans.add(PackingTestUtils.testContainerPlan(1));
    when(packingPlan.getContainers()).thenReturn(containerPlans);
    when(packing.pack()).thenReturn(packingPlan);
    LauncherUtils mockLauncherUtils = mock(LauncherUtils.class);
    when(mockLauncherUtils.createPackingPlan(any(Config.class), any(Config.class))).thenReturn(packingPlan);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    return runtime;
}

Example 59 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class LaunchRunnerTest method testSetTopologyFail.

@Test(expected = LauncherException.class)
public void testSetTopologyFail() throws Exception {
    Config runtime = createRunnerRuntime();
    Config config = createRunnerConfig();
    ILauncher launcher = Runtime.launcherClassInstance(runtime);
    LaunchRunner launchRunner = new LaunchRunner(config, runtime);
    SchedulerStateManagerAdaptor statemgr = Runtime.schedulerStateManagerAdaptor(runtime);
    when(statemgr.setTopology(any(TopologyAPI.Topology.class), eq(TOPOLOGY_NAME))).thenReturn(false);
    try {
        launchRunner.call();
    } finally {
        verify(launcher, never()).launch(any(PackingPlan.class));
    }
}

Example 60 with Config

use of com.twitter.heron.spi.common.Config in project incubator-heron by apache.

the class LaunchRunnerTest method testLaunchFailCleanUp.

@Test(expected = LauncherException.class)
public void testLaunchFailCleanUp() throws Exception {
    Config runtime = createRunnerRuntime();
    Config config = createRunnerConfig();
    ILauncher launcher = Runtime.launcherClassInstance(runtime);
    SchedulerStateManagerAdaptor statemgr = createTestSchedulerStateManager(runtime);
    LaunchRunner launchRunner = new LaunchRunner(config, runtime);
    when(launcher.launch(any(PackingPlan.class))).thenReturn(false);
    try {
        launchRunner.call();
    } finally {
        // Verify set && clean
        verify(statemgr).setTopology(any(TopologyAPI.Topology.class), eq(TOPOLOGY_NAME));
        verify(statemgr).setExecutionState(any(ExecutionEnvironment.ExecutionState.class), eq(TOPOLOGY_NAME));
        verify(statemgr).deleteExecutionState(eq(TOPOLOGY_NAME));
        verify(statemgr).deleteTopology(eq(TOPOLOGY_NAME));
    }
}