Examples with FsStateBackend org.apache.flink.runtime.state.filesystem.FsStateBackend used on opensource projects

Example 1 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class AbstractStateBackend method loadStateBackendFromConfig.

// ------------------------------------------------------------------------
//  Loading the state backend from a configuration 
// ------------------------------------------------------------------------
/**
	 * Loads the state backend from the configuration, from the parameter 'state.backend', as defined
	 * in {@link CoreOptions#STATE_BACKEND}.
	 * 
	 * <p>The state backends can be specified either via their shortcut name, or via the class name
	 * of a {@link StateBackendFactory}. If a StateBackendFactory class name is specified, the factory
	 * is instantiated (via its zero-argument constructor) and its
	 * {@link StateBackendFactory#createFromConfig(Configuration)} method is called.
	 *
	 * <p>Recognized shortcut names are '{@value AbstractStateBackend#MEMORY_STATE_BACKEND_NAME}',
	 * '{@value AbstractStateBackend#FS_STATE_BACKEND_NAME}', and
	 * '{@value AbstractStateBackend#ROCKSDB_STATE_BACKEND_NAME}'.
	 * 
	 * @param config The configuration to load the state backend from
	 * @param classLoader The class loader that should be used to load the state backend
	 * @param logger Optionally, a logger to log actions to (may be null)
	 * 
	 * @return The instantiated state backend.
	 * 
	 * @throws DynamicCodeLoadingException
	 *             Thrown if a state backend factory is configured and the factory class was not
	 *             found or the factory could not be instantiated
	 * @throws IllegalConfigurationException
	 *             May be thrown by the StateBackendFactory when creating / configuring the state
	 *             backend in the factory
	 * @throws IOException
	 *             May be thrown by the StateBackendFactory when instantiating the state backend
	 */
public static StateBackend loadStateBackendFromConfig(Configuration config, ClassLoader classLoader, @Nullable Logger logger) throws IllegalConfigurationException, DynamicCodeLoadingException, IOException {
    checkNotNull(config, "config");
    checkNotNull(classLoader, "classLoader");
    final String backendName = config.getString(CoreOptions.STATE_BACKEND);
    if (backendName == null) {
        return null;
    }
    // by default the factory class is the backend name 
    String factoryClassName = backendName;
    switch(backendName.toLowerCase()) {
        case MEMORY_STATE_BACKEND_NAME:
            if (logger != null) {
                logger.info("State backend is set to heap memory (checkpoint to JobManager)");
            }
            return new MemoryStateBackend();
        case FS_STATE_BACKEND_NAME:
            FsStateBackend fsBackend = new FsStateBackendFactory().createFromConfig(config);
            if (logger != null) {
                logger.info("State backend is set to heap memory (checkpoints to filesystem \"{}\")", fsBackend.getBasePath());
            }
            return fsBackend;
        case ROCKSDB_STATE_BACKEND_NAME:
            factoryClassName = "org.apache.flink.contrib.streaming.state.RocksDBStateBackendFactory";
        default:
            if (logger != null) {
                logger.info("Loading state backend via factory {}", factoryClassName);
            }
            StateBackendFactory<?> factory;
            try {
                @SuppressWarnings("rawtypes") Class<? extends StateBackendFactory> clazz = Class.forName(factoryClassName, false, classLoader).asSubclass(StateBackendFactory.class);
                factory = clazz.newInstance();
            } catch (ClassNotFoundException e) {
                throw new DynamicCodeLoadingException("Cannot find configured state backend factory class: " + backendName, e);
            } catch (ClassCastException | InstantiationException | IllegalAccessException e) {
                throw new DynamicCodeLoadingException("The class configured under '" + CoreOptions.STATE_BACKEND.key() + "' is not a valid state backend factory (" + backendName + ')', e);
            }
            return factory.createFromConfig(config);
    }
}

Example 2 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class FileStateBackendTest method testStateOutputStream.

@Test
public void testStateOutputStream() {
    URI basePath = randomHdfsFileUri();
    try {
        FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(basePath, 15));
        JobID jobId = new JobID();
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(jobId, "test_op");
        // we know how FsCheckpointStreamFactory is implemented so we know where it
        // will store checkpoints
        Path checkpointPath = new Path(new Path(basePath), jobId.toString());
        byte[] state1 = new byte[1274673];
        byte[] state2 = new byte[1];
        byte[] state3 = new byte[0];
        byte[] state4 = new byte[177];
        Random rnd = new Random();
        rnd.nextBytes(state1);
        rnd.nextBytes(state2);
        rnd.nextBytes(state3);
        rnd.nextBytes(state4);
        long checkpointId = 97231523452L;
        CheckpointStreamFactory.CheckpointStateOutputStream stream1 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream2 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream3 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream1.write(state1);
        stream2.write(state2);
        stream3.write(state3);
        FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
        ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
        ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();
        // use with try-with-resources
        FileStateHandle handle4;
        try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) {
            stream4.write(state4);
            handle4 = (FileStateHandle) stream4.closeAndGetHandle();
        }
        // close before accessing handle
        CheckpointStreamFactory.CheckpointStateOutputStream stream5 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream5.write(state4);
        stream5.close();
        try {
            stream5.closeAndGetHandle();
            fail();
        } catch (IOException e) {
        // uh-huh
        }
        validateBytesInStream(handle1.openInputStream(), state1);
        handle1.discardState();
        assertFalse(isDirectoryEmpty(checkpointPath));
        ensureFileDeleted(handle1.getFilePath());
        validateBytesInStream(handle2.openInputStream(), state2);
        handle2.discardState();
        // stream 3 has zero bytes, so it should not return anything
        assertNull(handle3);
        validateBytesInStream(handle4.openInputStream(), state4);
        handle4.discardState();
        assertTrue(isDirectoryEmpty(checkpointPath));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class RocksDBStateBackendTest method getStateBackend.

@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
    dbPath = tempFolder.newFolder().getAbsolutePath();
    String checkpointPath = tempFolder.newFolder().toURI().toString();
    RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath));
    backend.setDbStoragePath(dbPath);
    return backend;
}

Example 4 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class CassandraTupleWriteAheadSinkExample method main.

public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.enableCheckpointing(1000);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));
    CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource())).setQuery("INSERT INTO example.values (id, counter) values (?, ?);").enableWriteAheadLog().setClusterBuilder(new ClusterBuilder() {

        private static final long serialVersionUID = 2793938419775311824L;

        @Override
        public Cluster buildCluster(Cluster.Builder builder) {
            return builder.addContactPoint("127.0.0.1").build();
        }
    }).build();
    sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");
    env.execute();
}

Example 5 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class QsStateProducer method main.

public static void main(final String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    ParameterTool tool = ParameterTool.fromArgs(args);
    String tmpPath = tool.getRequired("tmp-dir");
    String stateBackendType = tool.getRequired("state-backend");
    StateBackend stateBackend;
    switch(stateBackendType) {
        case "rocksdb":
            stateBackend = new RocksDBStateBackend(tmpPath);
            break;
        case "fs":
            stateBackend = new FsStateBackend(tmpPath);
            break;
        case "memory":
            stateBackend = new MemoryStateBackend();
            break;
        default:
            throw new RuntimeException("Unsupported state backend " + stateBackendType);
    }
    env.setStateBackend(stateBackend);
    env.enableCheckpointing(1000L);
    env.getCheckpointConfig().setMaxConcurrentCheckpoints(1);
    env.getCheckpointConfig().setMinPauseBetweenCheckpoints(0);
    env.addSource(new EmailSource()).keyBy(new KeySelector<Email, String>() {

        private static final long serialVersionUID = -1480525724620425363L;

        @Override
        public String getKey(Email value) throws Exception {
            return QsConstants.KEY;
        }
    }).flatMap(new TestFlatMap());
    env.execute();
}