Examples with IllegalConfigurationException org.apache.flink.configuration.IllegalConfigurationException used on opensource projects

Example 11 with IllegalConfigurationException

use of org.apache.flink.configuration.IllegalConfigurationException in project flink by apache.

the class StreamingJobGraphGenerator method createJobGraph.

public JobGraph createJobGraph() {
    jobGraph = new JobGraph(streamGraph.getJobName());
    // make sure that all vertices start immediately
    jobGraph.setScheduleMode(ScheduleMode.EAGER);
    init();
    // Generate deterministic hashes for the nodes in order to identify them across
    // submission iff they didn't change.
    Map<Integer, byte[]> hashes = defaultStreamGraphHasher.traverseStreamGraphAndGenerateHashes(streamGraph);
    // Generate legacy version hashes for backwards compatibility
    List<Map<Integer, byte[]>> legacyHashes = new ArrayList<>(legacyStreamGraphHashers.size());
    for (StreamGraphHasher hasher : legacyStreamGraphHashers) {
        legacyHashes.add(hasher.traverseStreamGraphAndGenerateHashes(streamGraph));
    }
    setChaining(hashes, legacyHashes);
    setPhysicalEdges();
    setSlotSharing();
    configureCheckpointing();
    // set the ExecutionConfig last when it has been finalized
    try {
        jobGraph.setExecutionConfig(streamGraph.getExecutionConfig());
    } catch (IOException e) {
        throw new IllegalConfigurationException("Could not serialize the ExecutionConfig." + "This indicates that non-serializable types (like custom serializers) were registered");
    }
    return jobGraph;
}

Example 12 with IllegalConfigurationException

use of org.apache.flink.configuration.IllegalConfigurationException in project flink by apache.

the class FlinkYarnSessionCli method loadYarnPropertiesFile.

/**
	 * Tries to load a Flink Yarn properties file and returns the Yarn application id if successful
	 * @param cmdLine The command-line parameters
	 * @param flinkConfiguration The flink configuration
	 * @return Yarn application id or null if none could be retrieved
	 */
private String loadYarnPropertiesFile(CommandLine cmdLine, Configuration flinkConfiguration) {
    String jobManagerOption = cmdLine.getOptionValue(ADDRESS_OPTION.getOpt(), null);
    if (jobManagerOption != null) {
        // don't resume from properties file if a JobManager has been specified
        return null;
    }
    for (Option option : cmdLine.getOptions()) {
        if (ALL_OPTIONS.hasOption(option.getOpt())) {
            if (!option.getOpt().equals(DETACHED.getOpt())) {
                // don't resume from properties file if yarn options have been specified
                return null;
            }
        }
    }
    // load the YARN properties
    File propertiesFile = getYarnPropertiesLocation(flinkConfiguration);
    if (!propertiesFile.exists()) {
        return null;
    }
    logAndSysout("Found YARN properties file " + propertiesFile.getAbsolutePath());
    Properties yarnProperties = new Properties();
    try {
        try (InputStream is = new FileInputStream(propertiesFile)) {
            yarnProperties.load(is);
        }
    } catch (IOException e) {
        throw new RuntimeException("Cannot read the YARN properties file", e);
    }
    // get the Yarn application id from the properties file
    String applicationID = yarnProperties.getProperty(YARN_APPLICATION_ID_KEY);
    if (applicationID == null) {
        throw new IllegalConfigurationException("Yarn properties file found but doesn't contain a " + "Yarn application id. Please delete the file at " + propertiesFile.getAbsolutePath());
    }
    try {
        // try converting id to ApplicationId
        ConverterUtils.toApplicationId(applicationID);
    } catch (Exception e) {
        throw new RuntimeException("YARN properties contains an invalid entry for " + "application id: " + applicationID, e);
    }
    logAndSysout("Using Yarn application id from YARN properties " + applicationID);
    // configure the default parallelism from YARN
    String propParallelism = yarnProperties.getProperty(YARN_PROPERTIES_PARALLELISM);
    if (propParallelism != null) {
        // maybe the property is not set
        try {
            int parallelism = Integer.parseInt(propParallelism);
            flinkConfiguration.setInteger(ConfigConstants.DEFAULT_PARALLELISM_KEY, parallelism);
            logAndSysout("YARN properties set default parallelism to " + parallelism);
        } catch (NumberFormatException e) {
            throw new RuntimeException("Error while parsing the YARN properties: " + "Property " + YARN_PROPERTIES_PARALLELISM + " is not an integer.");
        }
    }
    // handle the YARN client's dynamic properties
    String dynamicPropertiesEncoded = yarnProperties.getProperty(YARN_PROPERTIES_DYNAMIC_PROPERTIES_STRING);
    Map<String, String> dynamicProperties = getDynamicProperties(dynamicPropertiesEncoded);
    for (Map.Entry<String, String> dynamicProperty : dynamicProperties.entrySet()) {
        flinkConfiguration.setString(dynamicProperty.getKey(), dynamicProperty.getValue());
    }
    return applicationID;
}

Example 13 with IllegalConfigurationException

use of org.apache.flink.configuration.IllegalConfigurationException in project flink by apache.

the class YarnHighAvailabilityServices method forYarnTaskManager.

/**
	 * Creates the high-availability services for the TaskManagers participating in
	 * a Flink YARN application.
	 *
	 * @param flinkConfig  The Flink configuration.
	 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
	 *
	 * @return The created high-availability services.
	 *
	 * @throws IOException Thrown, if the high-availability services could not be initialized.
	 */
public static YarnHighAvailabilityServices forYarnTaskManager(Configuration flinkConfig, org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {
    checkNotNull(flinkConfig, "flinkConfig");
    checkNotNull(hadoopConfig, "hadoopConfig");
    final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
    switch(mode) {
        case NONE:
            return new YarnPreConfiguredMasterNonHaServices(flinkConfig, hadoopConfig);
        case ZOOKEEPER:
            throw new UnsupportedOperationException("to be implemented");
        default:
            throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
    }
}

Example 14 with IllegalConfigurationException

use of org.apache.flink.configuration.IllegalConfigurationException in project flink by apache.

the class YarnHighAvailabilityServices method forSingleJobAppMaster.

// ------------------------------------------------------------------------
//  Factory from Configuration
// ------------------------------------------------------------------------
/**
	 * Creates the high-availability services for a single-job Flink YARN application, to be
	 * used in the Application Master that runs both ResourceManager and JobManager.
	 * 
	 * @param flinkConfig  The Flink configuration.
	 * @param hadoopConfig The Hadoop configuration for the YARN cluster.
	 * 
	 * @return The created high-availability services.
	 * 
	 * @throws IOException Thrown, if the high-availability services could not be initialized.
	 */
public static YarnHighAvailabilityServices forSingleJobAppMaster(Configuration flinkConfig, org.apache.hadoop.conf.Configuration hadoopConfig) throws IOException {
    checkNotNull(flinkConfig, "flinkConfig");
    checkNotNull(hadoopConfig, "hadoopConfig");
    final HighAvailabilityMode mode = HighAvailabilityMode.fromConfig(flinkConfig);
    switch(mode) {
        case NONE:
            return new YarnIntraNonHaMasterServices(flinkConfig, hadoopConfig);
        case ZOOKEEPER:
            throw new UnsupportedOperationException("to be implemented");
        default:
            throw new IllegalConfigurationException("Unrecognized high availability mode: " + mode);
    }
}

Example 15 with IllegalConfigurationException

use of org.apache.flink.configuration.IllegalConfigurationException in project flink by apache.

the class YarnClusterDescriptorTest method testConfigOverwrite.

@Test
public void testConfigOverwrite() {
    YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor();
    Configuration configuration = new Configuration();
    // overwrite vcores in config
    configuration.setInteger(ConfigConstants.YARN_VCORES, Integer.MAX_VALUE);
    clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));
    clusterDescriptor.setFlinkConfiguration(configuration);
    clusterDescriptor.setConfigurationDirectory(temporaryFolder.getRoot().getAbsolutePath());
    clusterDescriptor.setConfigurationFilePath(new Path(flinkConf.getPath()));
    // configure slots
    clusterDescriptor.setTaskManagerSlots(1);
    try {
        clusterDescriptor.deploy();
        fail("The deploy call should have failed.");
    } catch (RuntimeException e) {
        // we expect the cause to be an IllegalConfigurationException
        if (!(e.getCause() instanceof IllegalConfigurationException)) {
            throw e;
        }
    }
}