Examples with DelegatingStateBackend org.apache.flink.runtime.state.delegate.DelegatingStateBackend used on opensource projects

Example 1 with DelegatingStateBackend

use of org.apache.flink.runtime.state.delegate.DelegatingStateBackend in project flink by apache.

the class StateBackendLoader method loadChangelogStateBackend.

private static StateBackend loadChangelogStateBackend(StateBackend backend, ClassLoader classLoader) throws DynamicCodeLoadingException {
    // ChangelogStateBackend resides in a separate module, load it using reflection
    try {
        Constructor<? extends DelegatingStateBackend> constructor = Class.forName(CHANGELOG_STATE_BACKEND, false, classLoader).asSubclass(DelegatingStateBackend.class).getDeclaredConstructor(StateBackend.class);
        constructor.setAccessible(true);
        return constructor.newInstance(backend);
    } catch (ClassNotFoundException e) {
        throw new DynamicCodeLoadingException("Cannot find DelegateStateBackend class: " + CHANGELOG_STATE_BACKEND, e);
    } catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
        throw new DynamicCodeLoadingException("Fail to initialize: " + CHANGELOG_STATE_BACKEND, e);
    }
}

Example 2 with DelegatingStateBackend

use of org.apache.flink.runtime.state.delegate.DelegatingStateBackend in project flink by apache.

the class StateBackendLoader method fromApplicationOrConfigOrDefault.

/**
 * This is the state backend loader that loads a {@link DelegatingStateBackend} wrapping the
 * state backend loaded from {@link
 * StateBackendLoader#loadFromApplicationOrConfigOrDefaultInternal} when delegation is enabled.
 * If delegation is not enabled, the underlying wrapped state backend is returned instead.
 *
 * @param fromApplication StateBackend defined from application
 * @param isChangelogStateBackendEnableFromApplication whether to enable the
 *     ChangelogStateBackend from application
 * @param config The configuration to load the state backend from
 * @param classLoader The class loader that should be used to load the state backend
 * @param logger Optionally, a logger to log actions to (may be null)
 * @return The instantiated state backend.
 * @throws DynamicCodeLoadingException Thrown if a state backend (factory) is configured and the
 *     (factory) class was not found or could not be instantiated
 * @throws IllegalConfigurationException May be thrown by the StateBackendFactory when creating
 *     / configuring the state backend in the factory
 * @throws IOException May be thrown by the StateBackendFactory when instantiating the state
 *     backend
 */
public static StateBackend fromApplicationOrConfigOrDefault(@Nullable StateBackend fromApplication, TernaryBoolean isChangelogStateBackendEnableFromApplication, Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException {
    StateBackend rootBackend = loadFromApplicationOrConfigOrDefaultInternal(fromApplication, config, classLoader, logger);
    // Configuration from application will override the one from env.
    boolean enableChangeLog = TernaryBoolean.TRUE.equals(isChangelogStateBackendEnableFromApplication) || (TernaryBoolean.UNDEFINED.equals(isChangelogStateBackendEnableFromApplication) && config.get(StateChangelogOptions.ENABLE_STATE_CHANGE_LOG));
    StateBackend backend;
    if (enableChangeLog) {
        backend = loadChangelogStateBackend(rootBackend, classLoader);
        LOG.info("State backend loader loads {} to delegate {}", backend.getClass().getSimpleName(), rootBackend.getClass().getSimpleName());
    } else {
        backend = rootBackend;
        LOG.info("State backend loader loads the state backend as {}", backend.getClass().getSimpleName());
    }
    return backend;
}

Example 3 with DelegatingStateBackend

use of org.apache.flink.runtime.state.delegate.DelegatingStateBackend in project flink by apache.

the class CheckpointStorageLoader method load.

/**
 * Loads the configured {@link CheckpointStorage} for the job based on the following precedent
 * rules:
 *
 * <p>1) If the jobs configured {@link StateBackend} implements {@code CheckpointStorage} it
 * will always be used. This is to maintain backwards compatibility with older versions of Flink
 * that intermixed these responsibilities.
 *
 * <p>2) Use the {@link CheckpointStorage} instance configured via the {@code
 * StreamExecutionEnvironment}.
 *
 * <p>3) Use the {@link CheckpointStorage} instance configured via the clusters
 * <b>flink-conf.yaml</b>.
 *
 * <p>4) Load a default {@link CheckpointStorage} instance.
 *
 * @param fromApplication The checkpoint storage instance passed to the jobs
 *     StreamExecutionEnvironment. Or null if not was set.
 * @param configuredStateBackend The jobs configured state backend.
 * @param config The configuration to load the checkpoint storage from.
 * @param classLoader The class loader that should be used to load the checkpoint storage.
 * @param logger Optionally, a logger to log actions to (may be null).
 * @return The configured checkpoint storage instance.
 * @throws DynamicCodeLoadingException Thrown if a checkpoint storage factory is configured and
 *     the factory class was not found or the factory could not be instantiated
 * @throws IllegalConfigurationException May be thrown by the CheckpointStorageFactory when
 *     creating / configuring the checkpoint storage in the factory
 */
public static CheckpointStorage load(@Nullable CheckpointStorage fromApplication, @Nullable Path defaultSavepointDirectory, StateBackend configuredStateBackend, Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException {
    Preconditions.checkNotNull(config, "config");
    Preconditions.checkNotNull(classLoader, "classLoader");
    Preconditions.checkNotNull(configuredStateBackend, "statebackend");
    if (defaultSavepointDirectory != null) {
        // If a savepoint directory was manually specified in code
        // we override any value set in the flink-conf. This allows
        // us to pass this value to the CheckpointStorage instance
        // where it is needed at runtime while keeping its API logically
        // separated for users.
        config.set(CheckpointingOptions.SAVEPOINT_DIRECTORY, defaultSavepointDirectory.toString());
    }
    // Legacy state backends always take precedence for backwards compatibility.
    StateBackend rootStateBackend = (configuredStateBackend instanceof DelegatingStateBackend) ? ((DelegatingStateBackend) configuredStateBackend).getDelegatedStateBackend() : configuredStateBackend;
    if (rootStateBackend instanceof CheckpointStorage) {
        if (logger != null) {
            logger.info("Using legacy state backend {} as Job checkpoint storage", rootStateBackend);
            if (fromApplication != null) {
                logger.warn("Checkpoint storage passed via StreamExecutionEnvironment is ignored because legacy state backend '{}' is used. {}", rootStateBackend.getClass().getName(), LEGACY_PRECEDENCE_LOG_MESSAGE);
            }
            if (config.get(CheckpointingOptions.CHECKPOINT_STORAGE) != null) {
                logger.warn("Config option '{}' is ignored because legacy state backend '{}' is used. {}", CheckpointingOptions.CHECKPOINT_STORAGE.key(), rootStateBackend.getClass().getName(), LEGACY_PRECEDENCE_LOG_MESSAGE);
            }
        }
        return (CheckpointStorage) rootStateBackend;
    }
    if (fromApplication != null) {
        if (fromApplication instanceof ConfigurableCheckpointStorage) {
            if (logger != null) {
                logger.info("Using job/cluster config to configure application-defined checkpoint storage: {}", fromApplication);
                if (config.get(CheckpointingOptions.CHECKPOINT_STORAGE) != null) {
                    logger.warn("Config option '{}' is ignored because the checkpoint storage passed via StreamExecutionEnvironment takes precedence.", CheckpointingOptions.CHECKPOINT_STORAGE.key());
                }
            }
            return ((ConfigurableCheckpointStorage) fromApplication).configure(config, classLoader);
        }
        if (logger != null) {
            logger.info("Using application defined checkpoint storage: {}", fromApplication);
        }
        return fromApplication;
    }
    return fromConfig(config, classLoader, logger).orElseGet(() -> createDefaultCheckpointStorage(config, classLoader, logger));
}