Examples with HashMapStateBackendFactory org.apache.flink.runtime.state.hashmap.HashMapStateBackendFactory used on opensource projects

Example 1 with HashMapStateBackendFactory

use of org.apache.flink.runtime.state.hashmap.HashMapStateBackendFactory in project flink by apache.

the class StateBackendLoader method loadStateBackendFromConfig.

// ------------------------------------------------------------------------
// Loading the state backend from a configuration
// ------------------------------------------------------------------------
/**
 * Loads the unwrapped state backend from the configuration, from the parameter 'state.backend',
 * as defined in {@link StateBackendOptions#STATE_BACKEND}.
 *
 * <p>The state backends can be specified either via their shortcut name, or via the class name
 * of a {@link StateBackendFactory}. If a StateBackendFactory class name is specified, the
 * factory is instantiated (via its zero-argument constructor) and its {@link
 * StateBackendFactory#createFromConfig(ReadableConfig, ClassLoader)} method is called.
 *
 * <p>Recognized shortcut names are '{@value StateBackendLoader#HASHMAP_STATE_BACKEND_NAME}',
 * '{@value StateBackendLoader#ROCKSDB_STATE_BACKEND_NAME}' '{@value
 * StateBackendLoader#MEMORY_STATE_BACKEND_NAME}' (Deprecated), and '{@value
 * StateBackendLoader#FS_STATE_BACKEND_NAME}' (Deprecated).
 *
 * @param config The configuration to load the state backend from
 * @param classLoader The class loader that should be used to load the state backend
 * @param logger Optionally, a logger to log actions to (may be null)
 * @return The instantiated state backend.
 * @throws DynamicCodeLoadingException Thrown if a state backend factory is configured and the
 *     factory class was not found or the factory could not be instantiated
 * @throws IllegalConfigurationException May be thrown by the StateBackendFactory when creating
 *     / configuring the state backend in the factory
 * @throws IOException May be thrown by the StateBackendFactory when instantiating the state
 *     backend
 */
public static StateBackend loadStateBackendFromConfig(ReadableConfig config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException {
    checkNotNull(config, "config");
    checkNotNull(classLoader, "classLoader");
    final String backendName = config.get(StateBackendOptions.STATE_BACKEND);
    if (backendName == null) {
        return null;
    }
    // by default the factory class is the backend name
    String factoryClassName = backendName;
    switch(backendName.toLowerCase()) {
        case MEMORY_STATE_BACKEND_NAME:
            MemoryStateBackend backend = new MemoryStateBackendFactory().createFromConfig(config, classLoader);
            if (logger != null) {
                logger.warn("MemoryStateBackend has been deprecated. Please use 'hashmap' state " + "backend instead with JobManagerCheckpointStorage for equivalent " + "functionality");
                logger.info("State backend is set to job manager {}", backend);
            }
            return backend;
        case FS_STATE_BACKEND_NAME:
            if (logger != null) {
                logger.warn("{} state backend has been deprecated. Please use 'hashmap' state " + "backend instead.", backendName.toLowerCase());
            }
        // utilizes the same HeapKeyedStateBackend runtime implementation.
        case HASHMAP_STATE_BACKEND_NAME:
            HashMapStateBackend hashMapStateBackend = new HashMapStateBackendFactory().createFromConfig(config, classLoader);
            if (logger != null) {
                logger.info("State backend is set to heap memory {}", hashMapStateBackend);
            }
            return hashMapStateBackend;
        case ROCKSDB_STATE_BACKEND_NAME:
            factoryClassName = ROCKSDB_STATE_BACKEND_FACTORY;
        default:
            if (logger != null) {
                logger.info("Loading state backend via factory {}", factoryClassName);
            }
            StateBackendFactory<?> factory;
            try {
                @SuppressWarnings("rawtypes") Class<? extends StateBackendFactory> clazz = Class.forName(factoryClassName, false, classLoader).asSubclass(StateBackendFactory.class);
                factory = clazz.newInstance();
            } catch (ClassNotFoundException e) {
                throw new DynamicCodeLoadingException("Cannot find configured state backend factory class: " + backendName, e);
            } catch (ClassCastException | InstantiationException | IllegalAccessException e) {
                throw new DynamicCodeLoadingException("The class configured under '" + StateBackendOptions.STATE_BACKEND.key() + "' is not a valid state backend factory (" + backendName + ')', e);
            }
            return factory.createFromConfig(config, classLoader);
    }
}

Example 2 with HashMapStateBackendFactory

use of org.apache.flink.runtime.state.hashmap.HashMapStateBackendFactory in project flink by apache.

the class StateBackendLoader method loadFromApplicationOrConfigOrDefaultInternal.

/**
 * Checks if an application-defined state backend is given, and if not, loads the state backend
 * from the configuration, from the parameter 'state.backend', as defined in {@link
 * CheckpointingOptions#STATE_BACKEND}. If no state backend is configured, this instantiates the
 * default state backend (the {@link HashMapStateBackend}).
 *
 * <p>If an application-defined state backend is found, and the state backend is a {@link
 * ConfigurableStateBackend}, this methods calls {@link
 * ConfigurableStateBackend#configure(ReadableConfig, ClassLoader)} on the state backend.
 *
 * <p>Refer to {@link #loadStateBackendFromConfig(ReadableConfig, ClassLoader, Logger)} for
 * details on how the state backend is loaded from the configuration.
 *
 * @param config The configuration to load the state backend from
 * @param classLoader The class loader that should be used to load the state backend
 * @param logger Optionally, a logger to log actions to (may be null)
 * @return The instantiated state backend.
 * @throws DynamicCodeLoadingException Thrown if a state backend factory is configured and the
 *     factory class was not found or the factory could not be instantiated
 * @throws IllegalConfigurationException May be thrown by the StateBackendFactory when creating
 *     / configuring the state backend in the factory
 * @throws IOException May be thrown by the StateBackendFactory when instantiating the state
 *     backend
 */
private static StateBackend loadFromApplicationOrConfigOrDefaultInternal(@Nullable StateBackend fromApplication, Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException {
    checkNotNull(config, "config");
    checkNotNull(classLoader, "classLoader");
    final StateBackend backend;
    // (1) the application defined state backend has precedence
    if (fromApplication != null) {
        // see if this is supposed to pick up additional configuration parameters
        if (fromApplication instanceof ConfigurableStateBackend) {
            // needs to pick up configuration
            if (logger != null) {
                logger.info("Using job/cluster config to configure application-defined state backend: {}", fromApplication);
            }
            backend = ((ConfigurableStateBackend) fromApplication).configure(config, classLoader);
        } else {
            // keep as is!
            backend = fromApplication;
        }
        if (logger != null) {
            logger.info("Using application-defined state backend: {}", backend);
        }
    } else {
        // (2) check if the config defines a state backend
        final StateBackend fromConfig = loadStateBackendFromConfig(config, classLoader, logger);
        if (fromConfig != null) {
            backend = fromConfig;
        } else {
            // (3) use the default
            backend = new HashMapStateBackendFactory().createFromConfig(config, classLoader);
            if (logger != null) {
                logger.info("No state backend has been configured, using default (HashMap) {}", backend);
            }
        }
    }
    return backend;
}