Examples with RetryingRegistrationConfiguration org.apache.flink.runtime.registration.RetryingRegistrationConfiguration used on opensource projects

Example 1 with RetryingRegistrationConfiguration

use of org.apache.flink.runtime.registration.RetryingRegistrationConfiguration in project flink by apache.

the class TaskManagerServicesConfiguration method fromConfiguration.

// --------------------------------------------------------------------------------------------
// Parsing of Flink configuration
// --------------------------------------------------------------------------------------------
/**
 * Utility method to extract TaskManager config parameters from the configuration and to sanity
 * check them.
 *
 * @param configuration The configuration.
 * @param resourceID resource ID of the task manager
 * @param externalAddress identifying the IP address under which the TaskManager will be
 *     accessible
 * @param localCommunicationOnly True if only local communication is possible. Use only in cases
 *     where only one task manager runs.
 * @param taskExecutorResourceSpec resource specification of the TaskManager to start
 * @param workingDirectory working directory of the TaskManager
 * @return configuration of task manager services used to create them
 */
public static TaskManagerServicesConfiguration fromConfiguration(Configuration configuration, ResourceID resourceID, String externalAddress, boolean localCommunicationOnly, TaskExecutorResourceSpec taskExecutorResourceSpec, WorkingDirectory workingDirectory) throws Exception {
    String[] localStateRootDirs = ConfigurationUtils.parseLocalStateDirectories(configuration);
    final Reference<File[]> localStateDirs;
    if (localStateRootDirs.length == 0) {
        localStateDirs = Reference.borrowed(new File[] { workingDirectory.getLocalStateDirectory() });
    } else {
        File[] createdLocalStateDirs = new File[localStateRootDirs.length];
        final String localStateDirectoryName = LOCAL_STATE_SUB_DIRECTORY_ROOT + resourceID;
        for (int i = 0; i < localStateRootDirs.length; i++) {
            createdLocalStateDirs[i] = new File(localStateRootDirs[i], localStateDirectoryName);
        }
        localStateDirs = Reference.owned(createdLocalStateDirs);
    }
    boolean localRecoveryMode = configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY);
    final QueryableStateConfiguration queryableStateConfig = QueryableStateConfiguration.fromConfiguration(configuration);
    long timerServiceShutdownTimeout = configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION).toMillis();
    final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);
    final int externalDataPort = configuration.getInteger(NettyShuffleEnvironmentOptions.DATA_PORT);
    String bindAddr = configuration.getString(TaskManagerOptions.BIND_HOST, NetUtils.getWildcardIPAddress());
    InetAddress bindAddress = InetAddress.getByName(bindAddr);
    final String classLoaderResolveOrder = configuration.getString(CoreOptions.CLASSLOADER_RESOLVE_ORDER);
    final String[] alwaysParentFirstLoaderPatterns = CoreOptions.getParentFirstLoaderPatterns(configuration);
    final int numIoThreads = ClusterEntrypointUtils.getPoolSize(configuration);
    final String[] tmpDirs = ConfigurationUtils.parseTempDirectories(configuration);
    return new TaskManagerServicesConfiguration(configuration, resourceID, externalAddress, bindAddress, externalDataPort, localCommunicationOnly, tmpDirs, localStateDirs, localRecoveryMode, queryableStateConfig, ConfigurationParserUtils.getSlot(configuration), ConfigurationParserUtils.getPageSize(configuration), taskExecutorResourceSpec, timerServiceShutdownTimeout, retryingRegistrationConfiguration, ConfigurationUtils.getSystemResourceMetricsProbingInterval(configuration), FlinkUserCodeClassLoaders.ResolveOrder.fromString(classLoaderResolveOrder), alwaysParentFirstLoaderPatterns, numIoThreads);
}

Example 2 with RetryingRegistrationConfiguration

use of org.apache.flink.runtime.registration.RetryingRegistrationConfiguration in project flink by apache.

the class TaskManagerConfiguration method fromConfiguration.

// --------------------------------------------------------------------------------------------
// Static factory methods
// --------------------------------------------------------------------------------------------
public static TaskManagerConfiguration fromConfiguration(Configuration configuration, TaskExecutorResourceSpec taskExecutorResourceSpec, String externalAddress, File tmpWorkingDirectory) {
    int numberSlots = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, 1);
    if (numberSlots == -1) {
        numberSlots = 1;
    }
    final String[] tmpDirPaths = ConfigurationUtils.parseTempDirectories(configuration);
    final Time rpcTimeout = Time.fromDuration(configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION));
    LOG.debug("Messages have a max timeout of " + rpcTimeout);
    final Time slotTimeout = Time.milliseconds(configuration.get(TaskManagerOptions.SLOT_TIMEOUT).toMillis());
    Time finiteRegistrationDuration;
    try {
        Duration maxRegistrationDuration = configuration.get(TaskManagerOptions.REGISTRATION_TIMEOUT);
        finiteRegistrationDuration = Time.milliseconds(maxRegistrationDuration.toMillis());
    } catch (IllegalArgumentException e) {
        LOG.warn("Invalid format for parameter {}. Set the timeout to be infinite.", TaskManagerOptions.REGISTRATION_TIMEOUT.key());
        finiteRegistrationDuration = null;
    }
    final boolean exitOnOom = configuration.getBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY);
    final String taskManagerLogPath = configuration.getString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, System.getProperty("log.file"));
    final String taskManagerStdoutPath;
    final String taskManagerLogDir;
    if (taskManagerLogPath != null) {
        final int extension = taskManagerLogPath.lastIndexOf('.');
        taskManagerLogDir = new File(taskManagerLogPath).getParent();
        if (extension > 0) {
            taskManagerStdoutPath = taskManagerLogPath.substring(0, extension) + ".out";
        } else {
            taskManagerStdoutPath = null;
        }
    } else {
        taskManagerStdoutPath = null;
        taskManagerLogDir = null;
    }
    final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);
    return new TaskManagerConfiguration(numberSlots, TaskExecutorResourceUtils.generateDefaultSlotResourceProfile(taskExecutorResourceSpec, numberSlots), TaskExecutorResourceUtils.generateTotalAvailableResourceProfile(taskExecutorResourceSpec), tmpDirPaths, rpcTimeout, slotTimeout, finiteRegistrationDuration, configuration, exitOnOom, taskManagerLogPath, taskManagerStdoutPath, taskManagerLogDir, externalAddress, tmpWorkingDirectory, retryingRegistrationConfiguration);
}

Example 3 with RetryingRegistrationConfiguration

use of org.apache.flink.runtime.registration.RetryingRegistrationConfiguration in project flink by apache.

the class JobMasterConfiguration method fromConfiguration.

public static JobMasterConfiguration fromConfiguration(Configuration configuration) {
    final Time rpcTimeout = Time.fromDuration(configuration.get(AkkaOptions.ASK_TIMEOUT_DURATION));
    final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));
    final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];
    final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);
    return new JobMasterConfiguration(rpcTimeout, slotRequestTimeout, tmpDirectory, retryingRegistrationConfiguration, configuration);
}