Examples with FSDataOutputStream org.apache.flink.core.fs.FSDataOutputStream used on opensource projects

Example 1 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class MigrationV0ToV1Test method testSavepointMigrationV0ToV1.

/**
	 * Simple test of savepoint methods.
	 */
@Test
public void testSavepointMigrationV0ToV1() throws Exception {
    String target = tmp.getRoot().getAbsolutePath();
    assertEquals(0, tmp.getRoot().listFiles().length);
    long checkpointId = ThreadLocalRandom.current().nextLong(Integer.MAX_VALUE);
    int numTaskStates = 4;
    int numSubtaskStates = 16;
    Collection<org.apache.flink.migration.runtime.checkpoint.TaskState> expected = createTaskStatesOld(numTaskStates, numSubtaskStates);
    SavepointV0 savepoint = new SavepointV0(checkpointId, expected);
    assertEquals(SavepointV0.VERSION, savepoint.getVersion());
    assertEquals(checkpointId, savepoint.getCheckpointId());
    assertEquals(expected, savepoint.getOldTaskStates());
    assertFalse(savepoint.getOldTaskStates().isEmpty());
    Exception latestException = null;
    Path path = null;
    FSDataOutputStream fdos = null;
    FileSystem fs = null;
    try {
        // Try to create a FS output stream
        for (int attempt = 0; attempt < 10; attempt++) {
            path = new Path(target, FileUtils.getRandomFilename("savepoint-"));
            if (fs == null) {
                fs = FileSystem.get(path.toUri());
            }
            try {
                fdos = fs.create(path, false);
                break;
            } catch (Exception e) {
                latestException = e;
            }
        }
        if (fdos == null) {
            throw new IOException("Failed to create file output stream at " + path, latestException);
        }
        try (DataOutputStream dos = new DataOutputStream(fdos)) {
            dos.writeInt(SavepointStore.MAGIC_NUMBER);
            dos.writeInt(savepoint.getVersion());
            SavepointV0Serializer.INSTANCE.serializeOld(savepoint, dos);
        }
        ClassLoader cl = Thread.currentThread().getContextClassLoader();
        Savepoint sp = SavepointStore.loadSavepoint(path.toString(), cl);
        int t = 0;
        for (TaskState taskState : sp.getTaskStates()) {
            for (int p = 0; p < taskState.getParallelism(); ++p) {
                SubtaskState subtaskState = taskState.getState(p);
                ChainedStateHandle<StreamStateHandle> legacyOperatorState = subtaskState.getLegacyOperatorState();
                for (int c = 0; c < legacyOperatorState.getLength(); ++c) {
                    StreamStateHandle stateHandle = legacyOperatorState.get(c);
                    try (InputStream is = stateHandle.openInputStream()) {
                        Tuple4<Integer, Integer, Integer, Integer> expTestState = new Tuple4<>(0, t, p, c);
                        Tuple4<Integer, Integer, Integer, Integer> actTestState;
                        //check function state
                        if (p % 4 != 0) {
                            assertEquals(1, is.read());
                            actTestState = InstantiationUtil.deserializeObject(is, cl);
                            assertEquals(expTestState, actTestState);
                        } else {
                            assertEquals(0, is.read());
                        }
                        //check operator state
                        expTestState.f0 = 1;
                        actTestState = InstantiationUtil.deserializeObject(is, cl);
                        assertEquals(expTestState, actTestState);
                    }
                }
                //check keyed state
                KeyGroupsStateHandle keyGroupsStateHandle = subtaskState.getManagedKeyedState();
                if (t % 3 != 0) {
                    assertEquals(1, keyGroupsStateHandle.getNumberOfKeyGroups());
                    assertEquals(p, keyGroupsStateHandle.getGroupRangeOffsets().getKeyGroupRange().getStartKeyGroup());
                    ByteStreamStateHandle stateHandle = (ByteStreamStateHandle) keyGroupsStateHandle.getDelegateStateHandle();
                    HashMap<String, KvStateSnapshot<?, ?, ?, ?>> testKeyedState = MigrationInstantiationUtil.deserializeObject(stateHandle.getData(), cl);
                    assertEquals(2, testKeyedState.size());
                    for (KvStateSnapshot<?, ?, ?, ?> snapshot : testKeyedState.values()) {
                        MemValueState.Snapshot<?, ?, ?> castedSnapshot = (MemValueState.Snapshot<?, ?, ?>) snapshot;
                        byte[] data = castedSnapshot.getData();
                        assertEquals(t, data[0]);
                        assertEquals(p, data[1]);
                    }
                } else {
                    assertEquals(null, keyGroupsStateHandle);
                }
            }
            ++t;
        }
        savepoint.dispose();
    } finally {
        // Dispose
        SavepointStore.removeSavepointFile(path.toString());
    }
}

Example 2 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class FileCache method copy.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException {
    // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task.
    // we unwrap the file system to get raw streams without safety net
    FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri());
    FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri());
    if (!tFS.exists(targetPath)) {
        if (sFS.getFileStatus(sourcePath).isDir()) {
            tFS.mkdirs(targetPath);
            FileStatus[] contents = sFS.listStatus(sourcePath);
            for (FileStatus content : contents) {
                String distPath = content.getPath().toString();
                if (content.isDir()) {
                    if (distPath.endsWith("/")) {
                        distPath = distPath.substring(0, distPath.length() - 1);
                    }
                }
                String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/"));
                copy(content.getPath(), new Path(localPath), executable);
            }
        } else {
            try (FSDataOutputStream lfsOutput = tFS.create(targetPath, false);
                FSDataInputStream fsInput = sFS.open(sourcePath)) {
                IOUtils.copyBytes(fsInput, lfsOutput);
                //noinspection ResultOfMethodCallIgnored
                new File(targetPath.toString()).setExecutable(executable);
            } catch (IOException ioe) {
                LOG.error("could not copy file to local file cache.", ioe);
            }
        }
    }
}

Example 3 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class CsvBulkWriter method forPojo.

/**
 * Builds a writer based on a POJO class definition.
 *
 * @param pojoClass The class of the POJO.
 * @param stream The output stream.
 * @param <T> The type of the elements accepted by this writer.
 */
static <T> CsvBulkWriter<T, T, Void> forPojo(Class<T> pojoClass, FSDataOutputStream stream) {
    final Converter<T, T, Void> converter = (value, context) -> value;
    final CsvMapper csvMapper = new CsvMapper();
    final CsvSchema schema = csvMapper.schemaFor(pojoClass).withoutQuoteChar();
    return new CsvBulkWriter<>(csvMapper, schema, converter, null, stream);
}

Example 4 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class StateChangeFsUploader method upload.

private LocalResult upload(Path path, Collection<UploadTask> tasks) throws IOException {
    boolean wrappedStreamClosed = false;
    FSDataOutputStream fsStream = fileSystem.create(path, NO_OVERWRITE);
    try {
        fsStream.write(compression ? 1 : 0);
        try (OutputStreamWithPos stream = wrap(fsStream)) {
            final Map<UploadTask, Map<StateChangeSet, Long>> tasksOffsets = new HashMap<>();
            for (UploadTask task : tasks) {
                tasksOffsets.put(task, format.write(stream, task.changeSets));
            }
            FileStateHandle handle = new FileStateHandle(path, stream.getPos());
            // otherwise JM may receive invalid handles
            return new LocalResult(tasksOffsets, handle);
        } finally {
            wrappedStreamClosed = true;
        }
    } finally {
        if (!wrappedStreamClosed) {
            fsStream.close();
        }
    }
}

Example 5 with FSDataOutputStream

use of org.apache.flink.core.fs.FSDataOutputStream in project flink by apache.

the class FileOutputFormat method close.

@Override
public void close() throws IOException {
    final FSDataOutputStream s = this.stream;
    if (s != null) {
        this.stream = null;
        s.close();
    }
}