Examples with Config org.apache.heron.spi.common.Config used on opensource projects

Example 81 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class HealthManagerTest method testInitialize.

@Test
public void testInitialize() throws Exception {
    String topologyName = "testTopology";
    Config config = Config.newBuilder().put(Key.SCHEDULER_IS_SERVICE, true).put(Key.TOPOLOGY_NAME, topologyName).put(Key.ENVIRON, "environ").put(Key.CLUSTER, "cluster").build();
    SchedulerStateManagerAdaptor adaptor = mock(SchedulerStateManagerAdaptor.class);
    SchedulerLocation schedulerLocation = SchedulerLocation.newBuilder().setTopologyName(topologyName).setHttpEndpoint("http://127.0.0.1").build();
    when(adaptor.getSchedulerLocation(anyString())).thenReturn(schedulerLocation);
    HealthManagerMetrics publishingMetrics = mock(HealthManagerMetrics.class);
    AbstractModule baseModule = HealthManager.buildBaseModule("127.0.0.1", TrackerMetricsProvider.class.getName(), publishingMetrics);
    HealthManager healthManager = new HealthManager(config, baseModule);
    Map<String, Object> policy = new HashMap<>();
    policy.put(PolicyConfigKey.HEALTH_POLICY_CLASS.key(), TestPolicy.class.getName());
    policy.put("test-config", "test-value");
    String[] policyIds = { "policy" };
    HealthPolicyConfigReader policyConfigProvider = mock(HealthPolicyConfigReader.class);
    when(policyConfigProvider.getPolicyIds()).thenReturn(Arrays.asList(policyIds));
    when(policyConfigProvider.getPolicyConfig("policy")).thenReturn(policy);
    HealthManager spyHealthMgr = spy(healthManager);
    doReturn(adaptor).when(spyHealthMgr).createStateMgrAdaptor();
    doReturn(policyConfigProvider).when(spyHealthMgr).createPolicyConfigReader();
    spyHealthMgr.initialize();
    List<IHealthPolicy> healthPolicies = spyHealthMgr.getHealthPolicies();
    assertEquals(1, healthPolicies.size());
    TestPolicy healthPolicy = (TestPolicy) healthPolicies.iterator().next();
    assertNotNull(healthPolicy.schedulerClient);
    assertEquals(healthPolicy.stateMgrAdaptor, adaptor);
    assertNotNull(healthPolicy.metricsProvider);
    assertEquals(healthPolicy.config.getConfig("test-config"), "test-value");
}

Example 82 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class DownloadRunner method main.

// takes topology package URI and extracts it to a directory
public static void main(String[] args) throws Exception {
    CommandLineParser parser = new DefaultParser();
    Options slaManagerCliOptions = constructCliOptions();
    // parse the help options first.
    Options helpOptions = constructHelpOptions();
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(slaManagerCliOptions);
        return;
    }
    try {
        cmd = parser.parse(slaManagerCliOptions, args);
    } catch (ParseException e) {
        usage(slaManagerCliOptions);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
    DownloaderMode mode = DownloaderMode.cluster;
    if (cmd.hasOption(CliArgs.MODE.text)) {
        mode = DownloaderMode.valueOf(cmd.getOptionValue(CliArgs.MODE.text, null));
    }
    Config config;
    switch(mode) {
        case cluster:
            config = Config.toClusterMode(Config.newBuilder().putAll(ConfigLoader.loadClusterConfig()).build());
            break;
        case local:
            if (!cmd.hasOption(CliArgs.HERON_HOME.text) || !cmd.hasOption(CliArgs.CONFIG_PATH.text)) {
                throw new IllegalArgumentException("Missing heron_home or config_path argument");
            }
            String heronHome = cmd.getOptionValue(CliArgs.HERON_HOME.text, null);
            String configPath = cmd.getOptionValue(CliArgs.CONFIG_PATH.text, null);
            config = Config.toLocalMode(Config.newBuilder().putAll(ConfigLoader.loadConfig(heronHome, configPath, null, null)).build());
            break;
        default:
            throw new IllegalArgumentException("Invalid mode: " + cmd.getOptionValue(CliArgs.MODE.text));
    }
    String uri = cmd.getOptionValue(CliArgs.TOPOLOGY_PACKAGE_URI.text, null);
    String destination = cmd.getOptionValue(CliArgs.EXTRACT_DESTINATION.text, null);
    // make it compatible with old param format
    if (uri == null && destination == null) {
        String[] leftOverArgs = cmd.getArgs();
        if (leftOverArgs.length != 2) {
            System.err.println("Usage: downloader <topology-package-uri> <extract-destination>");
            return;
        }
        uri = leftOverArgs[0];
        destination = leftOverArgs[1];
    }
    final URI topologyLocation = new URI(uri);
    final Path topologyDestination = Paths.get(destination);
    final File file = topologyDestination.toFile();
    if (!file.exists()) {
        file.mkdirs();
    }
    Class clazz = Registry.UriToClass(config, topologyLocation);
    final Downloader downloader = Registry.getDownloader(clazz, topologyLocation);
    downloader.download(topologyLocation, topologyDestination);
}

Example 83 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class HealthManager method main.

public static void main(String[] args) throws Exception {
    CommandLineParser parser = new DefaultParser();
    Options slaManagerCliOptions = constructCliOptions();
    // parse the help options first.
    Options helpOptions = constructHelpOptions();
    CommandLine cmd = parser.parse(helpOptions, args, true);
    if (cmd.hasOption("h")) {
        usage(slaManagerCliOptions);
        return;
    }
    try {
        cmd = parser.parse(slaManagerCliOptions, args);
    } catch (ParseException e) {
        usage(slaManagerCliOptions);
        throw new RuntimeException("Error parsing command line options: ", e);
    }
    HealthManagerMode mode = HealthManagerMode.cluster;
    if (hasOption(cmd, CliArgs.MODE)) {
        mode = HealthManagerMode.valueOf(getOptionValue(cmd, CliArgs.MODE));
    }
    Config config;
    switch(mode) {
        case cluster:
            config = Config.toClusterMode(Config.newBuilder().putAll(ConfigLoader.loadClusterConfig()).putAll(commandLineConfigs(cmd)).build());
            break;
        case local:
            if (!hasOption(cmd, CliArgs.HERON_HOME) || !hasOption(cmd, CliArgs.CONFIG_PATH)) {
                throw new IllegalArgumentException("Missing heron_home or config_path argument");
            }
            String heronHome = getOptionValue(cmd, CliArgs.HERON_HOME);
            String configPath = getOptionValue(cmd, CliArgs.CONFIG_PATH);
            config = Config.toLocalMode(Config.newBuilder().putAll(ConfigLoader.loadConfig(heronHome, configPath, null, null)).putAll(commandLineConfigs(cmd)).build());
            break;
        default:
            throw new IllegalArgumentException("Invalid mode: " + getOptionValue(cmd, CliArgs.MODE));
    }
    setupLogging(cmd, config);
    LOG.fine(Arrays.toString(cmd.getOptions()));
    // Add the SystemConfig into SingletonRegistry
    SystemConfig systemConfig = SystemConfig.newBuilder(true).putAll(Context.systemFile(config), true).putAll(Context.overrideFile(config), true).build();
    SingletonRegistry.INSTANCE.registerSingleton(SystemConfig.HERON_SYSTEM_CONFIG, systemConfig);
    LOG.info("Static Heron config loaded successfully ");
    LOG.fine(config.toString());
    // load the default config value and override with any command line values
    String metricSourceClassName = config.getStringValue(PolicyConfigKey.METRIC_SOURCE_TYPE.key());
    metricSourceClassName = getOptionValue(cmd, CliArgs.METRIC_SOURCE_TYPE, metricSourceClassName);
    String metricsUrl = config.getStringValue(PolicyConfigKey.METRIC_SOURCE_URL.key());
    metricsUrl = getOptionValue(cmd, CliArgs.METRIC_SOURCE_URL, metricsUrl);
    // metrics reporting thread
    HealthManagerMetrics publishingMetrics = new HealthManagerMetrics(Integer.valueOf(getOptionValue(cmd, CliArgs.METRICSMGR_PORT)));
    AbstractModule module = buildBaseModule(metricsUrl, metricSourceClassName, publishingMetrics);
    HealthManager healthManager = new HealthManager(config, module);
    LOG.info("Initializing health manager");
    healthManager.initialize();
    LOG.info("Starting Health Manager");
    PoliciesExecutor policyExecutor = new PoliciesExecutor(healthManager.healthPolicies);
    ScheduledFuture<?> future = policyExecutor.start();
    LOG.info("Starting Health Manager metric posting thread");
    new Thread(publishingMetrics).start();
    try {
        future.get();
    } finally {
        policyExecutor.destroy();
        publishingMetrics.close();
    }
}

Example 84 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class AuroraLauncherTest method testLaunch.

@Test
public void testLaunch() throws Exception {
    Config config = Config.newBuilder().build();
    AuroraLauncher launcher = Mockito.spy(AuroraLauncher.class);
    launcher.initialize(config, config);
    LauncherUtils mockLauncherUtils = Mockito.mock(LauncherUtils.class);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    // Failed to schedule
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(false);
    Assert.assertFalse(launcher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(mockLauncherUtils).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    // Happy path
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(true);
    Assert.assertTrue(launcher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(mockLauncherUtils, Mockito.times(2)).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    launcher.close();
}

Example 85 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class KubernetesContextTest method createVolumeConfigs.

/**
 * Generate <code>Volume</code> Configs for testing.
 * @param testCases Test case container.
 *                  Input: [0] Config, [1] Boolean to indicate Manager/Executor.
 *                  Output: [0] expectedKeys, [1] expectedOptionsKeys, [2] expectedOptionsValues.
 * @param prefix Configuration prefix key to use in lookup.
 */
private void createVolumeConfigs(List<TestTuple<Pair<Config, Boolean>, Object[]>> testCases, String prefix) {
    final String keyPattern = prefix + "%%s.%%s";
    final String keyExecutor = String.format(keyPattern, KubernetesConstants.EXECUTOR_NAME);
    final String keyManager = String.format(keyPattern, KubernetesConstants.MANAGER_NAME);
    final String volumeNameOne = "volume-name-one";
    final String volumeNameTwo = "volume-name-two";
    final String claimName = "OnDeMaNd";
    final String storageClassField = VolumeConfigKeys.storageClassName.name();
    final String pathField = VolumeConfigKeys.path.name();
    final String claimNameField = VolumeConfigKeys.claimName.name();
    final String expectedStorageClass = "expected-storage-class";
    final String expectedPath = "/path/for/volume/expected";
    // Create test cases for Executor/Manager on even/odd indices respectively.
    for (int idx = 0; idx < 2; ++idx) {
        // Manager case is default.
        boolean isExecutor = false;
        String key = keyManager;
        String description = KubernetesConstants.MANAGER_NAME;
        // Executor case.
        if (idx % 2 == 0) {
            isExecutor = true;
            key = keyExecutor;
            description = KubernetesConstants.EXECUTOR_NAME;
        }
        final String storageClassKeyOne = String.format(key, volumeNameOne, storageClassField);
        final String storageClassKeyTwo = String.format(key, volumeNameTwo, storageClassField);
        final String pathKeyOne = String.format(key, volumeNameOne, pathField);
        final String pathKeyTwo = String.format(key, volumeNameTwo, pathField);
        final String claimNameKeyOne = String.format(key, volumeNameOne, claimNameField);
        final String claimNameKeyTwo = String.format(key, volumeNameTwo, claimNameField);
        final Config configPVC = Config.newBuilder().put(pathKeyOne, expectedPath).put(pathKeyTwo, expectedPath).put(claimNameKeyOne, claimName).put(claimNameKeyTwo, claimName).put(storageClassKeyOne, expectedStorageClass).put(storageClassKeyTwo, expectedStorageClass).build();
        final List<String> expectedKeys = Arrays.asList(volumeNameOne, volumeNameTwo);
        final List<VolumeConfigKeys> expectedOptionsKeys = Arrays.asList(VolumeConfigKeys.path, VolumeConfigKeys.storageClassName, VolumeConfigKeys.claimName);
        final List<String> expectedOptionsValues = Arrays.asList(expectedPath, expectedStorageClass, claimName);
        testCases.add(new TestTuple<>(description, new Pair<>(configPVC, isExecutor), new Object[] { expectedKeys, expectedOptionsKeys, expectedOptionsValues }));
        final Config configPVCDisabled = Config.newBuilder().put(KubernetesContext.KUBERNETES_VOLUME_FROM_CLI_DISABLED, "true").put(pathKeyOne, expectedPath).put(pathKeyTwo, expectedPath).put(claimNameKeyOne, claimName).put(claimNameKeyTwo, claimName).put(storageClassKeyOne, expectedStorageClass).put(storageClassKeyTwo, expectedStorageClass).build();
        testCases.add(new TestTuple<>(description + " Disabled should not error", new Pair<>(configPVCDisabled, !isExecutor), new Object[] { new LinkedList<String>(), new LinkedList<VolumeConfigKeys>(), new LinkedList<String>() }));
    }
}