Examples with Config org.apache.heron.spi.common.Config used on opensource projects

Example 46 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class TopologyResource method updateComponentParallelism.

protected Response updateComponentParallelism(String cluster, String role, String environment, String name, MultivaluedMap<String, String> params, List<String> components) {
    final List<Pair<String, Object>> keyValues = new ArrayList<>(Arrays.asList(Pair.create(Key.CLUSTER.value(), cluster), Pair.create(Key.ROLE.value(), role), Pair.create(Key.ENVIRON.value(), environment), Pair.create(Key.TOPOLOGY_NAME.value(), name), Pair.create(Keys.PARAM_COMPONENT_PARALLELISM, String.join(",", components))));
    // has a dry run been requested?
    if (params.containsKey(PARAM_DRY_RUN)) {
        keyValues.add(Pair.create(Key.DRY_RUN.value(), Boolean.TRUE));
    }
    final Set<Pair<String, Object>> overrides = getUpdateOverrides(params);
    // apply overrides if they exists
    if (!overrides.isEmpty()) {
        keyValues.addAll(overrides);
    }
    final Config config = createConfig(keyValues);
    getActionFactory().createRuntimeAction(config, ActionType.UPDATE).execute();
    return Response.ok().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(String.format("%s updated", name))).build();
}

Example 47 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class NomadSchedulerTest method setUp.

@Before
public void setUp() throws Exception {
    Config config = Config.newBuilder().put(Key.TOPOLOGY_NAME, TOPOLOGY_NAME).put(Key.TOPOLOGY_ID, TOPOLOGY_ID).put(NomadContext.HERON_NOMAD_SCHEDULER_URI, SCHEDULER_URI).put(NomadContext.HERON_NOMAD_CORE_FREQ_MAPPING, HERON_NOMAD_CORE_FREQ_MAPPING).put(Key.CORE_PACKAGE_URI, CORE_PACKAGE_URI).put(Key.USE_CORE_PACKAGE_URI, USE_CORE_PACKAGE_URI).put(Key.EXECUTOR_BINARY, EXECUTOR_BINARY).put(NomadContext.HERON_NOMAD_DRIVER, NomadConstants.NomadDriver.RAW_EXEC.getName()).put(NomadContext.HERON_NOMAD_NETWORK_MODE, "default").build();
    this.mockRuntime = config;
    this.mockConfig = config;
    scheduler = Mockito.spy(NomadScheduler.class);
}

Example 48 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class SlurmLauncherTest method testLaunch.

/**
 * Test slurm scheduler launcher
 */
@Test
public void testLaunch() throws Exception {
    Config config = createRunnerConfig();
    Config runtime = Mockito.mock(Config.class);
    PackingPlan packingPlan = Mockito.mock(PackingPlan.class);
    PackingPlan plan = new PackingPlan("plan.id", new HashSet<PackingPlan.ContainerPlan>());
    PowerMockito.spy(SlurmContext.class);
    PowerMockito.doReturn(WORKING_DIRECTORY).when(SlurmContext.class, "workingDirectory", config);
    LauncherUtils mockLauncherUtils = Mockito.mock(LauncherUtils.class);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    SlurmLauncher slurmLauncher = Mockito.spy(new SlurmLauncher());
    slurmLauncher.initialize(config, runtime);
    SlurmScheduler slurmScheduler = Mockito.spy(new SlurmScheduler());
    PowerMockito.doReturn(true).when(slurmScheduler).onSchedule(plan);
    // Failed to download
    Mockito.doReturn(false).when(slurmLauncher).setupWorkingDirectory();
    Assert.assertFalse(slurmLauncher.launch(packingPlan));
    Mockito.verify(slurmLauncher).setupWorkingDirectory();
    // Failed to schedule
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(false);
    PowerMockito.doReturn(true).when(slurmLauncher).setupWorkingDirectory();
    Assert.assertFalse(slurmLauncher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(mockLauncherUtils).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    // happy path
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(true);
    Assert.assertTrue(slurmLauncher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(slurmLauncher, Mockito.times(3)).launch(Mockito.any(PackingPlan.class));
    Mockito.verify(mockLauncherUtils, Mockito.times(2)).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    slurmLauncher.close();
}

Example 49 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class YarnLauncherTest method getHMDriverConfConstructsReefConfig.

// This test verifies if launcher correctly provides all heron specific configurations are to reef
// framework. It does so by ensuring required configs exist. The test fails if an unknown config
// is found. Presence of unknown config should be verified and then added to the test below
@Test
public void getHMDriverConfConstructsReefConfig() throws Exception {
    YarnLauncher launcher = new YarnLauncher();
    YarnLauncher spyLauncher = Mockito.spy(launcher);
    Mockito.doNothing().when(spyLauncher).addLibraryToClasspathSet(Mockito.anyString());
    // inputConf contains configs provided to the launcher
    Map<String, String> inputConf = new HashMap<>();
    // expected contains the reef configs launcher is expected to construct using testConfigs
    Map<String, String> expected = new HashMap<>();
    setConfigs(inputConf, expected, Key.TOPOLOGY_NAME, "topology", TopologyName.class);
    setConfigs(inputConf, expected, Key.TOPOLOGY_BINARY_FILE, "binary", TopologyJar.class);
    setConfigs(inputConf, expected, Key.TOPOLOGY_PACKAGE_FILE, "pack", TopologyPackageName.class);
    setConfigs(inputConf, expected, Key.CLUSTER, "cluster", Cluster.class);
    setConfigs(inputConf, expected, Key.ROLE, "role", Role.class);
    setConfigs(inputConf, expected, Key.ENVIRON, "env", Environ.class);
    setConfigs(inputConf, expected, YarnKey.HERON_SCHEDULER_YARN_QUEUE.value(), "q", JobQueue.class);
    setConfigs(inputConf, expected, YarnKey.YARN_SCHEDULER_DRIVER_MEMORY_MB.value(), "123", DriverMemory.class);
    setConfigs(inputConf, expected, Key.CORE_PACKAGE_URI, new File(".").getName(), HeronCorePackageName.class);
    // the following expected values are mostly specific to reef runtime
    expected.put(JobControlHandler.class.getSimpleName(), ClientManager.class.getName());
    expected.put(NodeDescriptorHandler.class.getSimpleName(), NodeDescriptorHandler.class.getName());
    expected.put(ResourceAllocationHandler.class.getSimpleName(), ResourceAllocationHandler.class.getName());
    expected.put(ResourceStatusHandler.class.getSimpleName(), ResourceStatusHandler.class.getName());
    expected.put(RuntimeParameters.RuntimeStatusHandler.class.getSimpleName(), ResourceManagerStatus.class.getName());
    expected.put(VerboseLogMode.class.getSimpleName(), "false");
    expected.put(HttpPort.class.getSimpleName(), "0");
    expected.put(DriverIdentifier.class.getSimpleName(), "topology");
    Config.Builder builder = new Config.Builder();
    for (String configName : inputConf.keySet()) {
        builder.put(configName, inputConf.get(configName));
    }
    builder.put(Key.STATE_MANAGER_CLASS, "statemanager");
    builder.put(Key.PACKING_CLASS, "packing");
    Config config = builder.build();
    spyLauncher.initialize(config, null);
    Configuration resultConf = spyLauncher.getHMDriverConf();
    for (NamedParameterNode<?> reefConfigNode : resultConf.getNamedParameters()) {
        String resultValue = resultConf.getNamedParameter(reefConfigNode);
        String expectedValue = expected.get(reefConfigNode.getName());
        if (expectedValue == null) {
            Assert.fail(String.format("Unknown config, %s:%s, provided to heron driver", reefConfigNode.getName(), resultConf.getNamedParameter(reefConfigNode)));
        } else {
            Assert.assertEquals(expectedValue, resultValue);
        }
        expected.remove(reefConfigNode.getName());
    }
    Assert.assertEquals(String.format("Missing expected configurations: %s", expected), 0, expected.size());
}

Example 50 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class V1ControllerTest method testAddVolumesIfPresent.

@Test
public void testAddVolumesIfPresent() {
    final String pathDefault = "config-host-volume-path";
    final String pathNameDefault = "config-host-volume-name";
    final Config configWithVolumes = Config.newBuilder().put(KubernetesContext.KUBERNETES_VOLUME_NAME, pathNameDefault).put(KubernetesContext.KUBERNETES_VOLUME_TYPE, Volumes.HOST_PATH).put(KubernetesContext.KUBERNETES_VOLUME_HOSTPATH_PATH, pathDefault).build();
    final V1Controller controllerWithVol = new V1Controller(configWithVolumes, RUNTIME);
    final V1Volume volumeDefault = new V1VolumeBuilder().withName(pathNameDefault).withNewHostPath().withNewPath(pathDefault).endHostPath().build();
    final V1Volume volumeToBeKept = new V1VolumeBuilder().withName("volume-to-be-kept-name").withNewHostPath().withNewPath("volume-to-be-kept-path").endHostPath().build();
    final List<V1Volume> customVolumeList = Arrays.asList(new V1VolumeBuilder().withName(pathNameDefault).withNewHostPath().withNewPath("this-path-must-be-replaced").endHostPath().build(), volumeToBeKept);
    final List<V1Volume> expectedDefault = Collections.singletonList(volumeDefault);
    final List<V1Volume> expectedCustom = Arrays.asList(volumeDefault, volumeToBeKept);
    // No Volumes set.
    V1Controller controllerDoNotSetVolumes = new V1Controller(Config.newBuilder().build(), RUNTIME);
    V1PodSpec podSpecNoSetVolumes = new V1PodSpec();
    controllerDoNotSetVolumes.addVolumesIfPresent(podSpecNoSetVolumes);
    Assert.assertNull(podSpecNoSetVolumes.getVolumes());
    // Default. Null Volumes.
    V1PodSpec podSpecNull = new V1PodSpecBuilder().build();
    controllerWithVol.addVolumesIfPresent(podSpecNull);
    Assert.assertTrue("Default VOLUMES should be set in container with null VOLUMES", CollectionUtils.containsAll(expectedDefault, podSpecNull.getVolumes()));
    // Empty Volumes list
    V1PodSpec podSpecEmpty = new V1PodSpecBuilder().withVolumes(new LinkedList<>()).build();
    controllerWithVol.addVolumesIfPresent(podSpecEmpty);
    Assert.assertTrue("Default VOLUMES should be set in container with empty VOLUMES", CollectionUtils.containsAll(expectedDefault, podSpecEmpty.getVolumes()));
    // Custom Volumes list
    V1PodSpec podSpecCustom = new V1PodSpecBuilder().withVolumes(customVolumeList).build();
    controllerWithVol.addVolumesIfPresent(podSpecCustom);
    Assert.assertTrue("Default VOLUMES should be set in container with custom VOLUMES", CollectionUtils.containsAll(expectedCustom, podSpecCustom.getVolumes()));
}