Examples with ClusterDeploymentException org.apache.flink.client.deployment.ClusterDeploymentException used on opensource projects

Example 1 with ClusterDeploymentException

use of org.apache.flink.client.deployment.ClusterDeploymentException in project flink by apache.

the class KubernetesClusterDescriptor method deployApplicationCluster.

@Override
public ClusterClientProvider<String> deployApplicationCluster(final ClusterSpecification clusterSpecification, final ApplicationConfiguration applicationConfiguration) throws ClusterDeploymentException {
    if (client.getService(KubernetesService.ServiceType.REST_SERVICE, clusterId).isPresent()) {
        throw new ClusterDeploymentException("The Flink cluster " + clusterId + " already exists.");
    }
    checkNotNull(clusterSpecification);
    checkNotNull(applicationConfiguration);
    final KubernetesDeploymentTarget deploymentTarget = KubernetesDeploymentTarget.fromConfig(flinkConfig);
    if (KubernetesDeploymentTarget.APPLICATION != deploymentTarget) {
        throw new ClusterDeploymentException("Couldn't deploy Kubernetes Application Cluster." + " Expected deployment.target=" + KubernetesDeploymentTarget.APPLICATION.getName() + " but actual one was \"" + deploymentTarget + "\"");
    }
    applicationConfiguration.applyToConfiguration(flinkConfig);
    // No need to do pipelineJars validation if it is a PyFlink job.
    if (!(PackagedProgramUtils.isPython(applicationConfiguration.getApplicationClassName()) || PackagedProgramUtils.isPython(applicationConfiguration.getProgramArguments()))) {
        final List<File> pipelineJars = KubernetesUtils.checkJarFileForApplicationMode(flinkConfig);
        Preconditions.checkArgument(pipelineJars.size() == 1, "Should only have one jar");
    }
    final ClusterClientProvider<String> clusterClientProvider = deployClusterInternal(KubernetesApplicationClusterEntrypoint.class.getName(), clusterSpecification, false);
    try (ClusterClient<String> clusterClient = clusterClientProvider.getClusterClient()) {
        LOG.info("Create flink application cluster {} successfully, JobManager Web Interface: {}", clusterId, clusterClient.getWebInterfaceURL());
    }
    return clusterClientProvider;
}

Example 2 with ClusterDeploymentException

use of org.apache.flink.client.deployment.ClusterDeploymentException in project flink by apache.

the class YarnClusterDescriptor method deployApplicationCluster.

@Override
public ClusterClientProvider<ApplicationId> deployApplicationCluster(final ClusterSpecification clusterSpecification, final ApplicationConfiguration applicationConfiguration) throws ClusterDeploymentException {
    checkNotNull(clusterSpecification);
    checkNotNull(applicationConfiguration);
    final YarnDeploymentTarget deploymentTarget = YarnDeploymentTarget.fromConfig(flinkConfiguration);
    if (YarnDeploymentTarget.APPLICATION != deploymentTarget) {
        throw new ClusterDeploymentException("Couldn't deploy Yarn Application Cluster." + " Expected deployment.target=" + YarnDeploymentTarget.APPLICATION.getName() + " but actual one was \"" + deploymentTarget.getName() + "\"");
    }
    applicationConfiguration.applyToConfiguration(flinkConfiguration);
    final List<String> pipelineJars = flinkConfiguration.getOptional(PipelineOptions.JARS).orElse(Collections.emptyList());
    Preconditions.checkArgument(pipelineJars.size() == 1, "Should only have one jar");
    try {
        return deployInternal(clusterSpecification, "Flink Application Cluster", YarnApplicationClusterEntryPoint.class.getName(), null, false);
    } catch (Exception e) {
        throw new ClusterDeploymentException("Couldn't deploy Yarn Application Cluster", e);
    }
}

Example 3 with ClusterDeploymentException

use of org.apache.flink.client.deployment.ClusterDeploymentException in project flink by apache.

the class KubernetesClusterDescriptor method deployClusterInternal.

private ClusterClientProvider<String> deployClusterInternal(String entryPoint, ClusterSpecification clusterSpecification, boolean detached) throws ClusterDeploymentException {
    final ClusterEntrypoint.ExecutionMode executionMode = detached ? ClusterEntrypoint.ExecutionMode.DETACHED : ClusterEntrypoint.ExecutionMode.NORMAL;
    flinkConfig.setString(ClusterEntrypoint.INTERNAL_CLUSTER_EXECUTION_MODE, executionMode.toString());
    flinkConfig.setString(KubernetesConfigOptionsInternal.ENTRY_POINT_CLASS, entryPoint);
    // Rpc, blob, rest, taskManagerRpc ports need to be exposed, so update them to fixed values.
    KubernetesUtils.checkAndUpdatePortConfigOption(flinkConfig, BlobServerOptions.PORT, Constants.BLOB_SERVER_PORT);
    KubernetesUtils.checkAndUpdatePortConfigOption(flinkConfig, TaskManagerOptions.RPC_PORT, Constants.TASK_MANAGER_RPC_PORT);
    KubernetesUtils.checkAndUpdatePortConfigOption(flinkConfig, RestOptions.BIND_PORT, Constants.REST_PORT);
    if (HighAvailabilityMode.isHighAvailabilityModeActivated(flinkConfig)) {
        flinkConfig.setString(HighAvailabilityOptions.HA_CLUSTER_ID, clusterId);
        KubernetesUtils.checkAndUpdatePortConfigOption(flinkConfig, HighAvailabilityOptions.HA_JOB_MANAGER_PORT_RANGE, flinkConfig.get(JobManagerOptions.PORT));
    }
    try {
        final KubernetesJobManagerParameters kubernetesJobManagerParameters = new KubernetesJobManagerParameters(flinkConfig, clusterSpecification);
        final FlinkPod podTemplate = kubernetesJobManagerParameters.getPodTemplateFilePath().map(file -> KubernetesUtils.loadPodFromTemplateFile(client, file, Constants.MAIN_CONTAINER_NAME)).orElse(new FlinkPod.Builder().build());
        final KubernetesJobManagerSpecification kubernetesJobManagerSpec = KubernetesJobManagerFactory.buildKubernetesJobManagerSpecification(podTemplate, kubernetesJobManagerParameters);
        client.createJobManagerComponent(kubernetesJobManagerSpec);
        return createClusterClientProvider(clusterId);
    } catch (Exception e) {
        try {
            LOG.warn("Failed to create the Kubernetes cluster \"{}\", try to clean up the residual resources.", clusterId);
            client.stopAndCleanupCluster(clusterId);
        } catch (Exception e1) {
            LOG.info("Failed to stop and clean up the Kubernetes cluster \"{}\".", clusterId, e1);
        }
        throw new ClusterDeploymentException("Could not create Kubernetes cluster \"" + clusterId + "\".", e);
    }
}

Example 4 with ClusterDeploymentException

use of org.apache.flink.client.deployment.ClusterDeploymentException in project flink by apache.

the class YarnClusterDescriptorTest method testFailIfTaskSlotsHigherThanMaxVcores.

@Test
public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentException {
    final Configuration flinkConfiguration = new Configuration();
    YarnClusterDescriptor clusterDescriptor = createYarnClusterDescriptor(flinkConfiguration);
    clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));
    try {
        clusterDescriptor.deploySessionCluster(clusterSpecification);
        fail("The deploy call should have failed.");
    } catch (ClusterDeploymentException e) {
        // we expect the cause to be an IllegalConfigurationException
        if (!(e.getCause() instanceof IllegalConfigurationException)) {
            throw e;
        }
    } finally {
        clusterDescriptor.close();
    }
}

Example 5 with ClusterDeploymentException

use of org.apache.flink.client.deployment.ClusterDeploymentException in project flink by apache.

the class YarnClusterDescriptorTest method testConfigOverwrite.

@Test
public void testConfigOverwrite() throws ClusterDeploymentException {
    Configuration configuration = new Configuration();
    // overwrite vcores in config
    configuration.setInteger(YarnConfigOptions.VCORES, Integer.MAX_VALUE);
    YarnClusterDescriptor clusterDescriptor = createYarnClusterDescriptor(configuration);
    clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));
    // configure slots
    ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder().createClusterSpecification();
    try {
        clusterDescriptor.deploySessionCluster(clusterSpecification);
        fail("The deploy call should have failed.");
    } catch (ClusterDeploymentException e) {
        // we expect the cause to be an IllegalConfigurationException
        if (!(e.getCause() instanceof IllegalConfigurationException)) {
            throw e;
        }
    } finally {
        clusterDescriptor.close();
    }
}