Examples with TopologySubmissionException org.apache.heron.scheduler.TopologySubmissionException used on opensource projects

Example 1 with TopologySubmissionException

use of org.apache.heron.scheduler.TopologySubmissionException in project heron by twitter.

the class V1Controller method getPodTemplateLocation.

/**
 * Extracts the <code>ConfigMap</code> and <code>Pod Template</code> names from the CLI parameter.
 * @param isExecutor Flag to indicate loading of <code>Pod Template</code> for <code>Executor</code>
 *                   or <code>Manager</code>.
 * @return A pair of the form <code>(ConfigMap, Pod Template)</code>.
 */
@VisibleForTesting
protected Pair<String, String> getPodTemplateLocation(boolean isExecutor) {
    final String podTemplateConfigMapName = KubernetesContext.getPodTemplateConfigMapName(getConfiguration(), isExecutor);
    if (podTemplateConfigMapName == null) {
        return null;
    }
    try {
        final int splitPoint = podTemplateConfigMapName.indexOf(".");
        final String configMapName = podTemplateConfigMapName.substring(0, splitPoint);
        final String podTemplateName = podTemplateConfigMapName.substring(splitPoint + 1);
        if (configMapName.isEmpty() || podTemplateName.isEmpty()) {
            throw new IllegalArgumentException("Empty ConfigMap or Pod Template name");
        }
        return new Pair<>(configMapName, podTemplateName);
    } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
        final String message = "Invalid ConfigMap and/or Pod Template name";
        KubernetesUtils.logExceptionWithDetails(LOG, message, e);
        throw new TopologySubmissionException(message);
    }
}

Example 2 with TopologySubmissionException

use of org.apache.heron.scheduler.TopologySubmissionException in project heron by twitter.

the class V1Controller method submit.

/**
 * Configures all components required by a <code>topology</code> and submits it to the Kubernetes scheduler.
 * @param packingPlan Used to configure the StatefulSets <code>Resource</code>s and replica count.
 * @return Success indicator.
 */
@Override
boolean submit(PackingPlan packingPlan) {
    final String topologyName = getTopologyName();
    if (!topologyName.equals(topologyName.toLowerCase())) {
        throw new TopologySubmissionException("K8S scheduler does not allow upper case topology's.");
    }
    final Resource containerResource = getContainerResource(packingPlan);
    final V1Service topologyService = createTopologyService();
    try {
        coreClient.createNamespacedService(getNamespace(), topologyService, null, null, null);
    } catch (ApiException e) {
        KubernetesUtils.logExceptionWithDetails(LOG, "Error creating topology service", e);
        throw new TopologySubmissionException(e.getMessage());
    }
    // Find the max number of instances in a container so that we can open
    // enough ports if remote debugging is enabled.
    int numberOfInstances = 0;
    for (PackingPlan.ContainerPlan containerPlan : packingPlan.getContainers()) {
        numberOfInstances = Math.max(numberOfInstances, containerPlan.getInstances().size());
    }
    final V1StatefulSet executors = createStatefulSet(containerResource, numberOfInstances, true);
    final V1StatefulSet manager = createStatefulSet(containerResource, numberOfInstances, false);
    try {
        appsClient.createNamespacedStatefulSet(getNamespace(), executors, null, null, null);
        appsClient.createNamespacedStatefulSet(getNamespace(), manager, null, null, null);
    } catch (ApiException e) {
        final String message = String.format("Error creating topology: %s%n", e.getResponseBody());
        KubernetesUtils.logExceptionWithDetails(LOG, message, e);
        throw new TopologySubmissionException(message);
    }
    return true;
}

Example 3 with TopologySubmissionException

use of org.apache.heron.scheduler.TopologySubmissionException in project heron by twitter.

the class V1Controller method configurePodSpec.

/**
 * Configures the <code>Pod Spec</code> section of the <code>StatefulSet</code>. The <code>Heron</code> container
 * will be configured to allow it to function but other supplied containers are loaded verbatim.
 * @param podTemplateSpec The <code>Pod Template Spec</code> section to update.
 * @param resource Passed down to configure the resource limits.
 * @param numberOfInstances Passed down to configure the ports.
 * @param isExecutor Flag used to configure components specific to <code>Executor</code> and <code>Manager</code>.
 * @param volumes <code>Volumes</code> generated from configurations options.
 * @param volumeMounts <code>Volume Mounts</code> generated from configurations options.
 */
private void configurePodSpec(final V1PodTemplateSpec podTemplateSpec, Resource resource, int numberOfInstances, boolean isExecutor, List<V1Volume> volumes, List<V1VolumeMount> volumeMounts) {
    if (podTemplateSpec.getSpec() == null) {
        podTemplateSpec.setSpec(new V1PodSpec());
    }
    final V1PodSpec podSpec = podTemplateSpec.getSpec();
    // Set the termination period to 0 so pods can be deleted quickly
    podSpec.setTerminationGracePeriodSeconds(0L);
    // Set the pod tolerations so pods are rescheduled when nodes go down
    // https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/#taint-based-evictions
    configureTolerations(podSpec);
    // Get <Heron> container and ignore all others.
    final String containerName = isExecutor ? KubernetesConstants.EXECUTOR_NAME : KubernetesConstants.MANAGER_NAME;
    V1Container heronContainer = null;
    List<V1Container> containers = podSpec.getContainers();
    if (containers != null) {
        for (V1Container container : containers) {
            final String name = container.getName();
            if (name != null && name.equals(containerName)) {
                if (heronContainer != null) {
                    throw new TopologySubmissionException(String.format("Multiple configurations found for '%s' container", containerName));
                }
                heronContainer = container;
            }
        }
    } else {
        containers = new LinkedList<>();
    }
    if (heronContainer == null) {
        heronContainer = new V1Container().name(containerName);
        containers.add(heronContainer);
    }
    if (!volumes.isEmpty() || !volumeMounts.isEmpty()) {
        configurePodWithVolumesAndMountsFromCLI(podSpec, heronContainer, volumes, volumeMounts);
    }
    configureHeronContainer(resource, numberOfInstances, heronContainer, isExecutor);
    podSpec.setContainers(containers);
    addVolumesIfPresent(podSpec);
    mountSecretsAsVolumes(podSpec);
}

Example 4 with TopologySubmissionException

use of org.apache.heron.scheduler.TopologySubmissionException in project heron by twitter.

the class KubernetesContext method getVolumeConfigs.

/**
 * Collects parameters form the <code>CLI</code> and generates a mapping between <code>Volumes</code>
 * and their configuration <code>key-value</code> pairs.
 * @param config Contains the configuration options collected from the <code>CLI</code>.
 * @param prefix Configuration key to lookup for options.
 * @param isExecutor Flag used to switch CLI commands for the <code>Executor</code> and <code>Manager</code>.
 * @return A mapping between <code>Volumes</code> and their configuration <code>key-value</code> pairs.
 * Will return an empty list if there are no Volume Claim Templates to be generated.
 */
@VisibleForTesting
protected static Map<String, Map<KubernetesConstants.VolumeConfigKeys, String>> getVolumeConfigs(final Config config, final String prefix, final boolean isExecutor) {
    final Logger LOG = Logger.getLogger(V1Controller.class.getName());
    final String prefixKey = String.format(prefix, isExecutor ? KubernetesConstants.EXECUTOR_NAME : KubernetesConstants.MANAGER_NAME);
    final Set<String> completeConfigParam = getConfigKeys(config, prefixKey);
    final int prefixLength = prefixKey.length();
    final int volumeNameIdx = 0;
    final int optionIdx = 1;
    final Matcher matcher = KubernetesConstants.VALID_LOWERCASE_RFC_1123_REGEX.matcher("");
    final Map<String, Map<KubernetesConstants.VolumeConfigKeys, String>> volumes = new HashMap<>();
    try {
        for (String param : completeConfigParam) {
            final String[] tokens = param.substring(prefixLength).split("\\.");
            final String volumeName = tokens[volumeNameIdx];
            final KubernetesConstants.VolumeConfigKeys key = KubernetesConstants.VolumeConfigKeys.valueOf(tokens[optionIdx]);
            final String value = config.getStringValue(param);
            Map<KubernetesConstants.VolumeConfigKeys, String> volume = volumes.get(volumeName);
            if (volume == null) {
                // Validate new Volume Names.
                if (!matcher.reset(volumeName).matches()) {
                    throw new TopologySubmissionException(String.format("Volume name `%s` does not match lowercase RFC-1123 pattern", volumeName));
                }
                volume = new HashMap<>();
                volumes.put(volumeName, volume);
            }
            volume.put(key, value);
        }
    } catch (IndexOutOfBoundsException | IllegalArgumentException e) {
        final String message = "Invalid Volume configuration option provided on CLI";
        LOG.log(Level.CONFIG, message);
        throw new TopologySubmissionException(message);
    }
    // All Volumes must contain a path.
    for (Map.Entry<String, Map<KubernetesConstants.VolumeConfigKeys, String>> volume : volumes.entrySet()) {
        final String path = volume.getValue().get(KubernetesConstants.VolumeConfigKeys.path);
        if (path == null || path.isEmpty()) {
            throw new TopologySubmissionException(String.format("Volume `%s`: All Volumes require a" + " 'path'.", volume.getKey()));
        }
    }
    // Check to see if functionality is disabled.
    if (KubernetesContext.getVolumesFromCLIDisabled(config) && !volumes.isEmpty()) {
        final String message = "Configuring Volumes from the CLI is disabled.";
        LOG.log(Level.WARNING, message);
        throw new TopologySubmissionException(message);
    }
    return volumes;
}

Example 5 with TopologySubmissionException

use of org.apache.heron.scheduler.TopologySubmissionException in project heron by twitter.

the class KubernetesContext method getVolumeClaimTemplates.

/**
 * Collects parameters form the <code>CLI</code> and validates options for <code>PVC</code>s.
 * @param config Contains the configuration options collected from the <code>CLI</code>.
 * @param isExecutor Flag used to collect CLI commands for the <code>Executor</code> and <code>Manager</code>.
 * @return A mapping between <code>Volumes</code> and their configuration <code>key-value</code> pairs.
 * Will return an empty list if there are no Volume Claim Templates to be generated.
 */
public static Map<String, Map<KubernetesConstants.VolumeConfigKeys, String>> getVolumeClaimTemplates(final Config config, final boolean isExecutor) {
    final Matcher matcher = KubernetesConstants.VALID_LOWERCASE_RFC_1123_REGEX.matcher("");
    final Map<String, Map<KubernetesConstants.VolumeConfigKeys, String>> volumes = getVolumeConfigs(config, KubernetesContext.KUBERNETES_VOLUME_CLAIM_PREFIX, isExecutor);
    for (Map.Entry<String, Map<KubernetesConstants.VolumeConfigKeys, String>> volume : volumes.entrySet()) {
        // Claim name is required.
        if (!volume.getValue().containsKey(KubernetesConstants.VolumeConfigKeys.claimName)) {
            throw new TopologySubmissionException(String.format("Volume `%s`: Persistent Volume" + " Claims require a `claimName`.", volume.getKey()));
        }
        for (Map.Entry<KubernetesConstants.VolumeConfigKeys, String> volumeConfig : volume.getValue().entrySet()) {
            final KubernetesConstants.VolumeConfigKeys key = volumeConfig.getKey();
            final String value = volumeConfig.getValue();
            switch(key) {
                case claimName:
                    // Claim names which are not OnDemand should be lowercase RFC-1123.
                    if (!matcher.reset(value).matches() && !KubernetesConstants.LABEL_ON_DEMAND.equalsIgnoreCase(value)) {
                        throw new TopologySubmissionException(String.format("Volume `%s`: `claimName` does" + " not match lowercase RFC-1123 pattern", volume.getKey()));
                    }
                    break;
                case storageClassName:
                    if (!matcher.reset(value).matches()) {
                        throw new TopologySubmissionException(String.format("Volume `%s`: `storageClassName`" + " does not match lowercase RFC-1123 pattern", volume.getKey()));
                    }
                    break;
                case sizeLimit:
                case accessModes:
                case volumeMode:
                case readOnly:
                case path:
                case subPath:
                    break;
                default:
                    throw new TopologySubmissionException(String.format("Volume `%s`: Invalid Persistent" + " Volume Claim type option for '%s'", volume.getKey(), key));
            }
        }
    }
    return volumes;
}