Examples with FileStateHandle org.apache.flink.runtime.state.filesystem.FileStateHandle used on opensource projects

Example 1 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class SavepointStoreTest method testStoreExternalizedCheckpointsToSameDirectory.

/**
	 * Tests that multiple externalized checkpoints can be stored to the same
	 * directory.
	 */
@Test
public void testStoreExternalizedCheckpointsToSameDirectory() throws Exception {
    String root = tmp.newFolder().getAbsolutePath();
    FileSystem fs = FileSystem.get(new Path(root).toUri());
    // Store
    SavepointV1 savepoint = new SavepointV1(1929292, SavepointV1Test.createTaskStates(4, 24));
    FileStateHandle store1 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
    fs.exists(store1.getFilePath());
    assertTrue(store1.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));
    FileStateHandle store2 = SavepointStore.storeExternalizedCheckpointToHandle(root, savepoint);
    fs.exists(store2.getFilePath());
    assertTrue(store2.getFilePath().getPath().contains(SavepointStore.EXTERNALIZED_CHECKPOINT_METADATA_FILE));
}

Example 2 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class FileStateBackendTest method testStateOutputStream.

@Test
public void testStateOutputStream() {
    URI basePath = randomHdfsFileUri();
    try {
        FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(basePath, 15));
        JobID jobId = new JobID();
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(jobId, "test_op");
        // we know how FsCheckpointStreamFactory is implemented so we know where it
        // will store checkpoints
        Path checkpointPath = new Path(new Path(basePath), jobId.toString());
        byte[] state1 = new byte[1274673];
        byte[] state2 = new byte[1];
        byte[] state3 = new byte[0];
        byte[] state4 = new byte[177];
        Random rnd = new Random();
        rnd.nextBytes(state1);
        rnd.nextBytes(state2);
        rnd.nextBytes(state3);
        rnd.nextBytes(state4);
        long checkpointId = 97231523452L;
        CheckpointStreamFactory.CheckpointStateOutputStream stream1 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream2 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream3 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream1.write(state1);
        stream2.write(state2);
        stream3.write(state3);
        FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
        ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
        ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();
        // use with try-with-resources
        FileStateHandle handle4;
        try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) {
            stream4.write(state4);
            handle4 = (FileStateHandle) stream4.closeAndGetHandle();
        }
        // close before accessing handle
        CheckpointStreamFactory.CheckpointStateOutputStream stream5 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream5.write(state4);
        stream5.close();
        try {
            stream5.closeAndGetHandle();
            fail();
        } catch (IOException e) {
        // uh-huh
        }
        validateBytesInStream(handle1.openInputStream(), state1);
        handle1.discardState();
        assertFalse(isDirectoryEmpty(checkpointPath));
        ensureFileDeleted(handle1.getFilePath());
        validateBytesInStream(handle2.openInputStream(), state2);
        handle2.discardState();
        // stream 3 has zero bytes, so it should not return anything
        assertNull(handle3);
        validateBytesInStream(handle4.openInputStream(), state4);
        handle4.discardState();
        assertTrue(isDirectoryEmpty(checkpointPath));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class SavepointStore method loadSavepointWithHandle.

/**
	 * Loads the savepoint at the specified path. This methods returns the savepoint, as well as the
	 * handle to the metadata.
	 *
	 * @param savepointFileOrDirectory Path to the parent savepoint directory or the meta data file.
	 * @param classLoader The class loader used to resolve serialized classes from legacy savepoint formats.
	 * @return The loaded savepoint
	 *
	 * @throws IOException Failures during load are forwarded
	 */
public static Tuple2<Savepoint, StreamStateHandle> loadSavepointWithHandle(String savepointFileOrDirectory, ClassLoader classLoader) throws IOException {
    checkNotNull(savepointFileOrDirectory, "savepointFileOrDirectory");
    checkNotNull(classLoader, "classLoader");
    Path path = new Path(savepointFileOrDirectory);
    LOG.info("Loading savepoint from {}", path);
    FileSystem fs = FileSystem.get(path.toUri());
    FileStatus status = fs.getFileStatus(path);
    // If this is a directory, we need to find the meta data file
    if (status.isDir()) {
        Path candidatePath = new Path(path, SAVEPOINT_METADATA_FILE);
        if (fs.exists(candidatePath)) {
            path = candidatePath;
            LOG.info("Using savepoint file in {}", path);
        } else {
            throw new IOException("Cannot find meta data file in directory " + path + ". Please try to load the savepoint directly from the meta data file " + "instead of the directory.");
        }
    }
    // load the savepoint
    final Savepoint savepoint;
    try (DataInputStream dis = new DataInputViewStreamWrapper(fs.open(path))) {
        int magicNumber = dis.readInt();
        if (magicNumber == MAGIC_NUMBER) {
            int version = dis.readInt();
            SavepointSerializer<?> serializer = SavepointSerializers.getSerializer(version);
            savepoint = serializer.deserialize(dis, classLoader);
        } else {
            throw new RuntimeException("Unexpected magic number. This can have multiple reasons: " + "(1) You are trying to load a Flink 1.0 savepoint, which is not supported by this " + "version of Flink. (2) The file you were pointing to is not a savepoint at all. " + "(3) The savepoint file has been corrupted.");
        }
    }
    // construct the stream handle to the metadata file
    // we get the size best-effort
    long size = 0;
    try {
        size = fs.getFileStatus(path).getLen();
    } catch (Exception ignored) {
    // we don't know the size, but we don't want to fail the savepoint loading for that
    }
    StreamStateHandle metadataHandle = new FileStateHandle(path, size);
    return new Tuple2<>(savepoint, metadataHandle);
}

Example 4 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class CheckpointCoordinatorTest method doTestPartitionableStateRepartitioning.

private void doTestPartitionableStateRepartitioning(Random r, int oldParallelism, int newParallelism, int numNamedStates, int maxPartitionsPerState) {
    List<List<OperatorStateHandle>> previousParallelOpInstanceStates = new ArrayList<>(oldParallelism);
    for (int i = 0; i < oldParallelism; ++i) {
        Path fakePath = new Path("/fake-" + i);
        Map<String, OperatorStateHandle.StateMetaInfo> namedStatesToOffsets = new HashMap<>();
        int off = 0;
        for (int s = 0; s < numNamedStates - 1; ++s) {
            long[] offs = new long[1 + r.nextInt(maxPartitionsPerState)];
            for (int o = 0; o < offs.length; ++o) {
                offs[o] = off;
                ++off;
            }
            OperatorStateHandle.Mode mode = r.nextInt(10) == 0 ? OperatorStateHandle.Mode.UNION : OperatorStateHandle.Mode.SPLIT_DISTRIBUTE;
            namedStatesToOffsets.put("State-" + s, new OperatorStateHandle.StateMetaInfo(offs, mode));
        }
        if (numNamedStates % 2 == 0) {
            // finally add a broadcast state
            long[] offs = { off + 1, off + 2, off + 3, off + 4 };
            namedStatesToOffsets.put("State-" + (numNamedStates - 1), new OperatorStateHandle.StateMetaInfo(offs, OperatorStateHandle.Mode.BROADCAST));
        }
        previousParallelOpInstanceStates.add(Collections.singletonList(new OperatorStreamStateHandle(namedStatesToOffsets, new FileStateHandle(fakePath, -1))));
    }
    Map<StreamStateHandle, Map<String, List<Long>>> expected = new HashMap<>();
    int taskIndex = 0;
    int expectedTotalPartitions = 0;
    for (List<OperatorStateHandle> previousParallelOpInstanceState : previousParallelOpInstanceStates) {
        Assert.assertEquals(1, previousParallelOpInstanceState.size());
        for (OperatorStateHandle psh : previousParallelOpInstanceState) {
            Map<String, OperatorStateHandle.StateMetaInfo> offsMap = psh.getStateNameToPartitionOffsets();
            Map<String, List<Long>> offsMapWithList = new HashMap<>(offsMap.size());
            for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> e : offsMap.entrySet()) {
                long[] offs = e.getValue().getOffsets();
                int replication;
                switch(e.getValue().getDistributionMode()) {
                    case UNION:
                        replication = newParallelism;
                        break;
                    case BROADCAST:
                        int extra = taskIndex < (newParallelism % oldParallelism) ? 1 : 0;
                        replication = newParallelism / oldParallelism + extra;
                        break;
                    case SPLIT_DISTRIBUTE:
                        replication = 1;
                        break;
                    default:
                        throw new RuntimeException("Unknown distribution mode " + e.getValue().getDistributionMode());
                }
                if (replication > 0) {
                    expectedTotalPartitions += replication * offs.length;
                    List<Long> offsList = new ArrayList<>(offs.length);
                    for (long off : offs) {
                        for (int p = 0; p < replication; ++p) {
                            offsList.add(off);
                        }
                    }
                    offsMapWithList.put(e.getKey(), offsList);
                }
            }
            if (!offsMapWithList.isEmpty()) {
                expected.put(psh.getDelegateStateHandle(), offsMapWithList);
            }
            taskIndex++;
        }
    }
    OperatorStateRepartitioner repartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
    List<List<OperatorStateHandle>> pshs = repartitioner.repartitionState(previousParallelOpInstanceStates, oldParallelism, newParallelism);
    Map<StreamStateHandle, Map<String, List<Long>>> actual = new HashMap<>();
    int minCount = Integer.MAX_VALUE;
    int maxCount = 0;
    int actualTotalPartitions = 0;
    for (int p = 0; p < newParallelism; ++p) {
        int partitionCount = 0;
        Collection<OperatorStateHandle> pshc = pshs.get(p);
        for (OperatorStateHandle sh : pshc) {
            for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> namedState : sh.getStateNameToPartitionOffsets().entrySet()) {
                Map<String, List<Long>> stateToOffsets = actual.get(sh.getDelegateStateHandle());
                if (stateToOffsets == null) {
                    stateToOffsets = new HashMap<>();
                    actual.put(sh.getDelegateStateHandle(), stateToOffsets);
                }
                List<Long> actualOffs = stateToOffsets.get(namedState.getKey());
                if (actualOffs == null) {
                    actualOffs = new ArrayList<>();
                    stateToOffsets.put(namedState.getKey(), actualOffs);
                }
                long[] add = namedState.getValue().getOffsets();
                for (long l : add) {
                    actualOffs.add(l);
                }
                partitionCount += namedState.getValue().getOffsets().length;
            }
        }
        minCount = Math.min(minCount, partitionCount);
        maxCount = Math.max(maxCount, partitionCount);
        actualTotalPartitions += partitionCount;
    }
    for (Map<String, List<Long>> v : actual.values()) {
        for (List<Long> l : v.values()) {
            Collections.sort(l);
        }
    }
    // possible.
    if (oldParallelism != newParallelism) {
        int maxLoadDiff = maxCount - minCount;
        Assert.assertTrue("Difference in partition load is > 1 : " + maxLoadDiff, maxLoadDiff <= 1);
    }
    Assert.assertEquals(expectedTotalPartitions, actualTotalPartitions);
    Assert.assertEquals(expected, actual);
}

Example 5 with FileStateHandle

use of org.apache.flink.runtime.state.filesystem.FileStateHandle in project flink by apache.

the class StateChangeFsUploader method upload.

private LocalResult upload(Path path, Collection<UploadTask> tasks) throws IOException {
    boolean wrappedStreamClosed = false;
    FSDataOutputStream fsStream = fileSystem.create(path, NO_OVERWRITE);
    try {
        fsStream.write(compression ? 1 : 0);
        try (OutputStreamWithPos stream = wrap(fsStream)) {
            final Map<UploadTask, Map<StateChangeSet, Long>> tasksOffsets = new HashMap<>();
            for (UploadTask task : tasks) {
                tasksOffsets.put(task, format.write(stream, task.changeSets));
            }
            FileStateHandle handle = new FileStateHandle(path, stream.getPos());
            // otherwise JM may receive invalid handles
            return new LocalResult(tasksOffsets, handle);
        } finally {
            wrappedStreamClosed = true;
        }
    } finally {
        if (!wrappedStreamClosed) {
            fsStream.close();
        }
    }
}