Examples with FSDataInputStream org.apache.flink.core.fs.FSDataInputStream used on opensource projects

Example 6 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class CheckpointCoordinatorTest method compareKeyedState.

public static void compareKeyedState(Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState, Collection<KeyGroupsStateHandle> actualPartitionedKeyGroupState) throws Exception {
    KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
    int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getNumberOfKeyGroups();
    int actualTotalKeyGroups = 0;
    for (KeyGroupsStateHandle keyGroupsStateHandle : actualPartitionedKeyGroupState) {
        actualTotalKeyGroups += keyGroupsStateHandle.getNumberOfKeyGroups();
    }
    assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);
    try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
        for (int groupId : expectedHeadOpKeyGroupStateHandle.keyGroups()) {
            long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
            inputStream.seek(offset);
            int expectedKeyGroupState = InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
            for (KeyGroupsStateHandle oneActualKeyGroupStateHandle : actualPartitionedKeyGroupState) {
                if (oneActualKeyGroupStateHandle.containsKeyGroup(groupId)) {
                    long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
                    try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
                        actualInputStream.seek(actualOffset);
                        int actualGroupState = InstantiationUtil.deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
                        assertEquals(expectedKeyGroupState, actualGroupState);
                    }
                }
            }
        }
    }
}

Example 7 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class KeyedOneInputStreamOperatorTestHarness method restore.

/**
	 *
	 */
@Override
public void restore(StreamStateHandle snapshot) throws Exception {
    try (FSDataInputStream inStream = snapshot.openInputStream()) {
        if (operator instanceof StreamCheckpointedOperator) {
            ((StreamCheckpointedOperator) operator).restoreState(inStream);
        }
        byte keyedStatePresent = (byte) inStream.read();
        if (keyedStatePresent == 1) {
            ObjectInputStream ois = new ObjectInputStream(inStream);
            this.restoredKeyedState = Collections.singletonList((KeyGroupsStateHandle) ois.readObject());
        }
    }
}

Example 8 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class FileCache method copy.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException {
    // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task.
    // we unwrap the file system to get raw streams without safety net
    FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri());
    FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri());
    if (!tFS.exists(targetPath)) {
        if (sFS.getFileStatus(sourcePath).isDir()) {
            tFS.mkdirs(targetPath);
            FileStatus[] contents = sFS.listStatus(sourcePath);
            for (FileStatus content : contents) {
                String distPath = content.getPath().toString();
                if (content.isDir()) {
                    if (distPath.endsWith("/")) {
                        distPath = distPath.substring(0, distPath.length() - 1);
                    }
                }
                String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/"));
                copy(content.getPath(), new Path(localPath), executable);
            }
        } else {
            try (FSDataOutputStream lfsOutput = tFS.create(targetPath, false);
                FSDataInputStream fsInput = sFS.open(sourcePath)) {
                IOUtils.copyBytes(fsInput, lfsOutput);
                //noinspection ResultOfMethodCallIgnored
                new File(targetPath.toString()).setExecutable(executable);
            } catch (IOException ioe) {
                LOG.error("could not copy file to local file cache.", ioe);
            }
        }
    }
}

Example 9 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class MultiStreamStateHandleTest method testRandomRead.

@Test
public void testRandomRead() throws IOException {
    MultiStreamStateHandle multiStreamStateHandle = new MultiStreamStateHandle(streamStateHandles);
    try (FSDataInputStream in = multiStreamStateHandle.openInputStream()) {
        for (int i = 0; i < 1000; ++i) {
            int pos = random.nextInt(TEST_DATA_LENGTH);
            int readLen = random.nextInt(TEST_DATA_LENGTH);
            in.seek(pos);
            while (--readLen > 0 && pos < TEST_DATA_LENGTH) {
                assertEquals(pos, in.getPos());
                assertEquals(testData[pos++], in.read());
            }
        }
        in.seek(TEST_DATA_LENGTH);
        assertEquals(TEST_DATA_LENGTH, in.getPos());
        assertEquals(-1, in.read());
        try {
            in.seek(TEST_DATA_LENGTH + 1);
            fail();
        } catch (Exception ignored) {
        }
    }
}

Example 10 with FSDataInputStream

use of org.apache.flink.core.fs.FSDataInputStream in project flink by apache.

the class MultiStreamStateHandleTest method testEmptyList.

@Test
public void testEmptyList() throws IOException {
    MultiStreamStateHandle multiStreamStateHandle = new MultiStreamStateHandle(Collections.<StreamStateHandle>emptyList());
    try (FSDataInputStream in = multiStreamStateHandle.openInputStream()) {
        assertEquals(0, in.getPos());
        in.seek(0);
        assertEquals(0, in.getPos());
        assertEquals(-1, in.read());
        try {
            in.seek(1);
            fail();
        } catch (Exception ignored) {
        }
    }
}