Examples with FsStateBackend org.apache.flink.runtime.state.filesystem.FsStateBackend used on opensource projects

Example 11 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class TaskManagerProcessFailureStreamingRecoveryITCase method testTaskManagerFailure.

@Override
public void testTaskManagerFailure(Configuration configuration, final File coordinateDir) throws Exception {
    final File tempCheckpointDir = tempFolder.newFolder();
    StreamExecutionEnvironment env = StreamExecutionEnvironment.createRemoteEnvironment("localhost", // not needed since we use ZooKeeper
    1337, configuration);
    env.setParallelism(PARALLELISM);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.enableCheckpointing(200);
    env.setStateBackend(new FsStateBackend(tempCheckpointDir.getAbsoluteFile().toURI()));
    DataStream<Long> result = env.addSource(new SleepyDurableGenerateSequence(coordinateDir, DATA_COUNT)).map(new MapFunction<Long, Long>() {

        @Override
        public Long map(Long value) throws Exception {
            return value;
        }
    }).startNewChain().map(new Mapper(coordinateDir));
    // write result to temporary file
    result.addSink(new CheckpointedSink(DATA_COUNT));
    // blocking call until execution is done
    env.execute();
}

Example 12 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class RocksDBStateBackendConfigTest method testRocksDbReconfigurationCopiesExistingValues.

// ------------------------------------------------------------------------
// Reconfiguration
// ------------------------------------------------------------------------
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
    final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
    final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();
    final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);
    // these must not be the default options
    final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
    assertNotEquals(predOptions, original.getPredefinedOptions());
    original.setPredefinedOptions(predOptions);
    final RocksDBOptionsFactory optionsFactory = mock(RocksDBOptionsFactory.class);
    original.setRocksDBOptions(optionsFactory);
    final String[] localDirs = new String[] { tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
    original.setDbStoragePaths(localDirs);
    RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());
    assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
    assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
    assertEquals(original.getRocksDBOptions(), copy.getRocksDBOptions());
    assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());
    FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
    assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
    assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}

Example 13 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class EventTimeWindowCheckpointingITCase method getConfiguration.

private Configuration getConfiguration() throws Exception {
    // print a message when starting a test method to avoid Travis' <tt>"Maven produced no
    // output for xxx seconds."</tt> messages
    System.out.println("Starting " + getClass().getCanonicalName() + "#" + name.getMethodName() + ".");
    // Testing HA Scenario / ZKCompletedCheckpointStore with incremental checkpoints
    StateBackendEnum stateBackendEnum = getStateBackend();
    if (ROCKSDB_INCREMENTAL_ZK.equals(stateBackendEnum)) {
        zkServer = new TestingServer();
        zkServer.start();
    }
    Configuration config = createClusterConfig();
    config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_PER_CHANNEL, buffersPerChannel);
    switch(stateBackendEnum) {
        case MEM:
            this.stateBackend = new MemoryStateBackend(MAX_MEM_STATE_SIZE);
            break;
        case FILE:
            {
                final File backups = tempFolder.newFolder().getAbsoluteFile();
                this.stateBackend = new FsStateBackend(Path.fromLocalFile(backups));
                break;
            }
        case ROCKSDB_FULL:
            {
                setupRocksDB(config, -1, false);
                break;
            }
        case ROCKSDB_INCREMENTAL:
            // Test RocksDB based timer service as well
            config.set(RocksDBOptions.TIMER_SERVICE_FACTORY, EmbeddedRocksDBStateBackend.PriorityQueueStateType.ROCKSDB);
            setupRocksDB(config, 16, true);
            break;
        case ROCKSDB_INCREMENTAL_ZK:
            {
                setupRocksDB(config, 16, true);
                break;
            }
        default:
            throw new IllegalStateException("No backend selected.");
    }
    // Configure DFS DSTL for this test as it might produce too much GC pressure if
    // ChangelogStateBackend is used.
    // Doing it on cluster level unconditionally as randomization currently happens on the job
    // level (environment); while this factory can only be set on the cluster level.
    FsStateChangelogStorageFactory.configure(config, tempFolder.newFolder());
    return config;
}

Example 14 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class EventTimeWindowCheckpointingITCase method setupRocksDB.

private void setupRocksDB(Configuration config, int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
    // Configure the managed memory size as 64MB per slot for rocksDB state backend.
    config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.ofMebiBytes(PARALLELISM / NUM_OF_TASK_MANAGERS * 64));
    final String rocksDb = tempFolder.newFolder().getAbsolutePath();
    final File backups = tempFolder.newFolder().getAbsoluteFile();
    // we use the fs backend with small threshold here to test the behaviour with file
    // references, not self contained byte handles
    RocksDBStateBackend rdb = new RocksDBStateBackend(new FsStateBackend(Path.fromLocalFile(backups).toUri(), fileSizeThreshold), incrementalCheckpoints);
    rdb.setDbStoragePath(rocksDb);
    this.stateBackend = rdb;
}

Example 15 with FsStateBackend

use of org.apache.flink.runtime.state.filesystem.FsStateBackend in project flink by apache.

the class FileStateBackendTest method testStateOutputStream.

@Test
public void testStateOutputStream() throws IOException {
    File basePath = tempFolder.newFolder().getAbsoluteFile();
    try {
        // the state backend has a very low in-mem state threshold (15 bytes)
        FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(basePath.toURI(), 15));
        JobID jobId = new JobID();
        // we know how FsCheckpointStreamFactory is implemented so we know where it
        // will store checkpoints
        File checkpointPath = new File(basePath.getAbsolutePath(), jobId.toString());
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(jobId, "test_op");
        byte[] state1 = new byte[1274673];
        byte[] state2 = new byte[1];
        byte[] state3 = new byte[0];
        byte[] state4 = new byte[177];
        Random rnd = new Random();
        rnd.nextBytes(state1);
        rnd.nextBytes(state2);
        rnd.nextBytes(state3);
        rnd.nextBytes(state4);
        long checkpointId = 97231523452L;
        CheckpointStreamFactory.CheckpointStateOutputStream stream1 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream2 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream3 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream1.write(state1);
        stream2.write(state2);
        stream3.write(state3);
        FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
        ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
        ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();
        // use with try-with-resources
        StreamStateHandle handle4;
        try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) {
            stream4.write(state4);
            handle4 = stream4.closeAndGetHandle();
        }
        // close before accessing handle
        CheckpointStreamFactory.CheckpointStateOutputStream stream5 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream5.write(state4);
        stream5.close();
        try {
            stream5.closeAndGetHandle();
            fail();
        } catch (IOException e) {
        // uh-huh
        }
        validateBytesInStream(handle1.openInputStream(), state1);
        handle1.discardState();
        assertFalse(isDirectoryEmpty(basePath));
        ensureLocalFileDeleted(handle1.getFilePath());
        validateBytesInStream(handle2.openInputStream(), state2);
        handle2.discardState();
        // nothing was written to the stream, so it will return nothing
        assertNull(handle3);
        validateBytesInStream(handle4.openInputStream(), state4);
        handle4.discardState();
        assertTrue(isDirectoryEmpty(checkpointPath));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}