Examples with Config org.apache.heron.spi.common.Config used on opensource projects

Example 91 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class KubernetesContextTest method createVolumeEmptyDirError.

/**
 * Create test cases for <code>Empty Directory</code> errors.
 * @param testCases Test case container.
 *                  Input: [0] Config, [1] Boolean to indicate Manager/Executor.
 *                  Output: Error message
 * @param isExecutor Boolean to indicate Manager/Executor test case generation.
 */
private void createVolumeEmptyDirError(List<TestTuple<Pair<Config, Boolean>, String>> testCases, boolean isExecutor) {
    final String volumeNameValid = "volume-name-valid";
    final String passingValue = "should-pass";
    final String failureValue = "Should-Fail";
    final String processName = isExecutor ? KubernetesConstants.EXECUTOR_NAME : KubernetesConstants.MANAGER_NAME;
    final String keyPattern = String.format(KubernetesContext.KUBERNETES_VOLUME_EMPTYDIR_PREFIX + "%%s.%%s", processName);
    // Medium is invalid.
    final Config configInvalidMedium = Config.newBuilder().put(String.format(keyPattern, volumeNameValid, "path"), passingValue).put(String.format(keyPattern, volumeNameValid, "sizeLimit"), passingValue).put(String.format(keyPattern, volumeNameValid, "medium"), failureValue).build();
    testCases.add(new TestTuple<>(processName + ": Invalid 'medium' should trigger exception", new Pair<>(configInvalidMedium, isExecutor), "must be 'Memory' or empty."));
    // Invalid option.
    final Config configInvalidOption = Config.newBuilder().put(String.format(keyPattern, volumeNameValid, "path"), passingValue).put(String.format(keyPattern, volumeNameValid, "subPath"), passingValue).put(String.format(keyPattern, volumeNameValid, "sizeLimit"), passingValue).put(String.format(keyPattern, volumeNameValid, "medium"), "Memory").put(String.format(keyPattern, volumeNameValid, "accessModes"), passingValue).build();
    testCases.add(new TestTuple<>(processName + ": Invalid option should trigger exception", new Pair<>(configInvalidOption, isExecutor), "Directory type option"));
}

Example 92 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class V1Controller method mountVolumeIfPresent.

/**
 * Adds volume mounts to the <code>container</code> that Heron requires. Heron's values taking precedence.
 * @param container <code>container</code> to be configured.
 */
@VisibleForTesting
protected void mountVolumeIfPresent(final V1Container container) {
    final Config config = getConfiguration();
    if (KubernetesContext.hasContainerVolume(config)) {
        final V1VolumeMount mount = new V1VolumeMount().name(KubernetesContext.getContainerVolumeName(config)).mountPath(KubernetesContext.getContainerVolumeMountPath(config));
        // Merge volume mounts. Deduplicate using mount's name with Heron defaults taking precedence.
        KubernetesUtils.V1ControllerUtils<V1VolumeMount> utils = new KubernetesUtils.V1ControllerUtils<>();
        container.setVolumeMounts(utils.mergeListsDedupe(Collections.singletonList(mount), container.getVolumeMounts(), Comparator.comparing(V1VolumeMount::getName), "Pod Template Volume Mounts"));
    }
}

Example 93 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class V1Controller method getExecutorCommand.

/**
 * Generates the command to start Heron within the <code>container</code>.
 * @param containerId Passed down to <>SchedulerUtils</> to generate executor command.
 * @param numOfInstances Used to configure the debugging ports.
 * @param isExecutor Flag used to generate the correct <code>shard_id</code>.
 * @return The complete command to start Heron in a <code>container</code>.
 */
protected List<String> getExecutorCommand(String containerId, int numOfInstances, boolean isExecutor) {
    final Config configuration = getConfiguration();
    final Config runtimeConfiguration = getRuntimeConfiguration();
    final Map<ExecutorPort, String> ports = KubernetesConstants.EXECUTOR_PORTS.entrySet().stream().collect(Collectors.toMap(Map.Entry::getKey, e -> e.getValue().toString()));
    if (TopologyUtils.getTopologyRemoteDebuggingEnabled(Runtime.topology(runtimeConfiguration)) && numOfInstances != 0) {
        List<String> remoteDebuggingPorts = new LinkedList<>();
        IntStream.range(0, numOfInstances).forEach(i -> {
            int port = KubernetesConstants.JVM_REMOTE_DEBUGGER_PORT + i;
            remoteDebuggingPorts.add(String.valueOf(port));
        });
        ports.put(ExecutorPort.JVM_REMOTE_DEBUGGER_PORTS, String.join(",", remoteDebuggingPorts));
    }
    final String[] executorCommand = SchedulerUtils.getExecutorCommand(configuration, runtimeConfiguration, containerId, ports);
    return Arrays.asList("sh", "-c", KubernetesUtils.getConfCommand(configuration) + " && " + KubernetesUtils.getFetchCommand(configuration, runtimeConfiguration) + " && " + setShardIdEnvironmentVariableCommand(isExecutor) + " && " + String.join(" ", executorCommand));
}

Example 94 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class LauncherUtilsTest method constructsConfigWithTopologyInfo.

@Test
public void constructsConfigWithTopologyInfo() throws Exception {
    TopologyAPI.Topology mockTopology = PowerMockito.mock(TopologyAPI.Topology.class);
    PowerMockito.when(mockTopology.getId()).thenReturn("testTopologyId");
    PowerMockito.when(mockTopology.getName()).thenReturn("testTopologyName");
    SchedulerStateManagerAdaptor mockStMgr = Mockito.mock(SchedulerStateManagerAdaptor.class);
    PowerMockito.spy(TopologyUtils.class);
    PowerMockito.doReturn(456).when(TopologyUtils.class, "getNumContainers", mockTopology);
    Config runtime = Config.newBuilder().putAll(LauncherUtils.getInstance().createPrimaryRuntime(mockTopology)).putAll(LauncherUtils.getInstance().createAdaptorRuntime(mockStMgr)).build();
    Assert.assertEquals("testTopologyId", Runtime.topologyId(runtime));
    Assert.assertEquals("testTopologyName", Runtime.topologyName(runtime));
    Assert.assertEquals(mockTopology, Runtime.topology(runtime));
    Assert.assertEquals(mockStMgr, Runtime.schedulerStateManagerAdaptor(runtime));
    Assert.assertEquals(456 + 1, Runtime.numContainers(runtime).longValue());
}

Example 95 with Config

use of org.apache.heron.spi.common.Config in project heron by twitter.

the class KubernetesLauncherTest method testLaunch.

@Test
public void testLaunch() throws Exception {
    Config config = Config.newBuilder().build();
    KubernetesLauncher launcher = Mockito.spy(KubernetesLauncher.class);
    launcher.initialize(config, config);
    LauncherUtils mockLauncherUtils = Mockito.mock(LauncherUtils.class);
    PowerMockito.spy(LauncherUtils.class);
    PowerMockito.doReturn(mockLauncherUtils).when(LauncherUtils.class, "getInstance");
    // Launched successfully
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(true);
    Assert.assertTrue(launcher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(mockLauncherUtils).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    // Failed to launch
    Mockito.when(mockLauncherUtils.onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class))).thenReturn(false);
    Assert.assertFalse(launcher.launch(Mockito.mock(PackingPlan.class)));
    Mockito.verify(mockLauncherUtils, Mockito.times(2)).onScheduleAsLibrary(Mockito.any(Config.class), Mockito.any(Config.class), Mockito.any(IScheduler.class), Mockito.any(PackingPlan.class));
    launcher.close();
}