Examples with MemoryStateBackend org.apache.flink.runtime.state.memory.MemoryStateBackend used on opensource projects

Example 1 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class AbstractStateBackend method loadStateBackendFromConfig.

// ------------------------------------------------------------------------
//  Loading the state backend from a configuration 
// ------------------------------------------------------------------------
/**
	 * Loads the state backend from the configuration, from the parameter 'state.backend', as defined
	 * in {@link CoreOptions#STATE_BACKEND}.
	 * 
	 * <p>The state backends can be specified either via their shortcut name, or via the class name
	 * of a {@link StateBackendFactory}. If a StateBackendFactory class name is specified, the factory
	 * is instantiated (via its zero-argument constructor) and its
	 * {@link StateBackendFactory#createFromConfig(Configuration)} method is called.
	 *
	 * <p>Recognized shortcut names are '{@value AbstractStateBackend#MEMORY_STATE_BACKEND_NAME}',
	 * '{@value AbstractStateBackend#FS_STATE_BACKEND_NAME}', and
	 * '{@value AbstractStateBackend#ROCKSDB_STATE_BACKEND_NAME}'.
	 * 
	 * @param config The configuration to load the state backend from
	 * @param classLoader The class loader that should be used to load the state backend
	 * @param logger Optionally, a logger to log actions to (may be null)
	 * 
	 * @return The instantiated state backend.
	 * 
	 * @throws DynamicCodeLoadingException
	 *             Thrown if a state backend factory is configured and the factory class was not
	 *             found or the factory could not be instantiated
	 * @throws IllegalConfigurationException
	 *             May be thrown by the StateBackendFactory when creating / configuring the state
	 *             backend in the factory
	 * @throws IOException
	 *             May be thrown by the StateBackendFactory when instantiating the state backend
	 */
public static StateBackend loadStateBackendFromConfig(Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException {
    checkNotNull(config, "config");
    checkNotNull(classLoader, "classLoader");
    final String backendName = config.getString(CoreOptions.STATE_BACKEND);
    if (backendName == null) {
        return null;
    }
    // by default the factory class is the backend name 
    String factoryClassName = backendName;
    switch(backendName.toLowerCase()) {
        case MEMORY_STATE_BACKEND_NAME:
            if (logger != null) {
                logger.info("State backend is set to heap memory (checkpoint to JobManager)");
            }
            return new MemoryStateBackend();
        case FS_STATE_BACKEND_NAME:
            FsStateBackend fsBackend = new FsStateBackendFactory().createFromConfig(config);
            if (logger != null) {
                logger.info("State backend is set to heap memory (checkpoints to filesystem \"{}\")", fsBackend.getBasePath());
            }
            return fsBackend;
        case ROCKSDB_STATE_BACKEND_NAME:
            factoryClassName = "org.apache.flink.contrib.streaming.state.RocksDBStateBackendFactory";
        default:
            if (logger != null) {
                logger.info("Loading state backend via factory {}", factoryClassName);
            }
            StateBackendFactory<?> factory;
            try {
                @SuppressWarnings("rawtypes") Class<? extends StateBackendFactory> clazz = Class.forName(factoryClassName, false, classLoader).asSubclass(StateBackendFactory.class);
                factory = clazz.newInstance();
            } catch (ClassNotFoundException e) {
                throw new DynamicCodeLoadingException("Cannot find configured state backend factory class: " + backendName, e);
            } catch (ClassCastException | InstantiationException | IllegalAccessException e) {
                throw new DynamicCodeLoadingException("The class configured under '" + CoreOptions.STATE_BACKEND.key() + "' is not a valid state backend factory (" + backendName + ')', e);
            }
            return factory.createFromConfig(config);
    }
}

Example 2 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class StateBackendLoadingTest method testInstantiateMemoryBackendByDefault.

@Test
public void testInstantiateMemoryBackendByDefault() throws Exception {
    StateBackend backend = AbstractStateBackend.loadStateBackendFromConfigOrCreateDefault(new Configuration(), cl, null);
    assertTrue(backend instanceof MemoryStateBackend);
}

Example 3 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class StateBackendLoadingTest method testLoadMemoryStateBackend.

@Test
public void testLoadMemoryStateBackend() throws Exception {
    // we configure with the explicit string (rather than AbstractStateBackend#X_STATE_BACKEND_NAME)
    // to guard against config-breaking changes of the name 
    final Configuration config = new Configuration();
    config.setString(backendKey, "jobmanager");
    StateBackend backend = AbstractStateBackend.loadStateBackendFromConfigOrCreateDefault(new Configuration(), cl, null);
    assertTrue(backend instanceof MemoryStateBackend);
}

Example 4 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class MemoryStateBackendTest method testOversizedState.

@Test
public void testOversizedState() {
    try {
        MemoryStateBackend backend = new MemoryStateBackend(10);
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(new JobID(), "test_op");
        HashMap<String, Integer> state = new HashMap<>();
        state.put("hey there", 2);
        state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);
        try {
            CheckpointStreamFactory.CheckpointStateOutputStream outStream = streamFactory.createCheckpointStateOutputStream(12, 459);
            ObjectOutputStream oos = new ObjectOutputStream(outStream);
            oos.writeObject(state);
            oos.flush();
            outStream.closeAndGetHandle();
            fail("this should cause an exception");
        } catch (IOException e) {
        // now darling, isn't that exactly what we wanted?
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 5 with MemoryStateBackend

use of org.apache.flink.runtime.state.memory.MemoryStateBackend in project flink by apache.

the class ManualWindowSpeedITCase method testTumblingIngestionTimeWindowsWithRocksDBBackend.

@Test
public void testTumblingIngestionTimeWindowsWithRocksDBBackend() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.IngestionTime);
    env.setParallelism(1);
    env.setStateBackend(new RocksDBStateBackend(new MemoryStateBackend()));
    env.addSource(new InfiniteTupleSource(10_000)).keyBy(0).timeWindow(Time.seconds(3)).reduce(new ReduceFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, Integer> reduce(Tuple2<String, Integer> value1, Tuple2<String, Integer> value2) throws Exception {
            return Tuple2.of(value1.f0, value1.f1 + value2.f1);
        }
    }).filter(new FilterFunction<Tuple2<String, Integer>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public boolean filter(Tuple2<String, Integer> value) throws Exception {
            return value.f0.startsWith("Tuple 0");
        }
    }).print();
    env.execute();
}