Examples with FileStatus org.apache.flink.core.fs.FileStatus used on opensource projects

Example 1 with FileStatus

use of org.apache.flink.core.fs.FileStatus in project flink by apache.

the class PythonPlanBinderTest method findTestFiles.

private static List<String> findTestFiles() throws Exception {
    List<String> files = new ArrayList<>();
    FileSystem fs = FileSystem.getLocalFileSystem();
    FileStatus[] status = fs.listStatus(new Path(fs.getWorkingDirectory().toString() + "/src/test/python/org/apache/flink/python/api"));
    for (FileStatus f : status) {
        String file = f.getPath().toString();
        if (file.endsWith(".py")) {
            files.add(file);
        }
    }
    return files;
}

Example 2 with FileStatus

use of org.apache.flink.core.fs.FileStatus in project flink by apache.

the class FileInputFormat method getStatistics.

/**
	 * Obtains basic file statistics containing only file size. If the input is a directory, then the size is the sum of all contained files.
	 * 
	 * @see org.apache.flink.api.common.io.InputFormat#getStatistics(org.apache.flink.api.common.io.statistics.BaseStatistics)
	 */
@Override
public FileBaseStatistics getStatistics(BaseStatistics cachedStats) throws IOException {
    final FileBaseStatistics cachedFileStats = (cachedStats != null && cachedStats instanceof FileBaseStatistics) ? (FileBaseStatistics) cachedStats : null;
    try {
        final Path path = this.filePath;
        final FileSystem fs = FileSystem.get(path.toUri());
        return getFileStats(cachedFileStats, path, fs, new ArrayList<FileStatus>(1));
    } catch (IOException ioex) {
        if (LOG.isWarnEnabled()) {
            LOG.warn("Could not determine statistics for file '" + this.filePath + "' due to an io error: " + ioex.getMessage());
        }
    } catch (Throwable t) {
        if (LOG.isErrorEnabled()) {
            LOG.error("Unexpected problem while getting the file statistics for file '" + this.filePath + "': " + t.getMessage(), t);
        }
    }
    // no statistics available
    return null;
}

Example 3 with FileStatus

use of org.apache.flink.core.fs.FileStatus in project flink by apache.

the class SavepointStore method loadSavepointWithHandle.

/**
	 * Loads the savepoint at the specified path. This methods returns the savepoint, as well as the
	 * handle to the metadata.
	 *
	 * @param savepointFileOrDirectory Path to the parent savepoint directory or the meta data file.
	 * @param classLoader The class loader used to resolve serialized classes from legacy savepoint formats.
	 * @return The loaded savepoint
	 *
	 * @throws IOException Failures during load are forwarded
	 */
public static Tuple2<Savepoint, StreamStateHandle> loadSavepointWithHandle(String savepointFileOrDirectory, ClassLoader classLoader) throws IOException {
    checkNotNull(savepointFileOrDirectory, "savepointFileOrDirectory");
    checkNotNull(classLoader, "classLoader");
    Path path = new Path(savepointFileOrDirectory);
    LOG.info("Loading savepoint from {}", path);
    FileSystem fs = FileSystem.get(path.toUri());
    FileStatus status = fs.getFileStatus(path);
    // If this is a directory, we need to find the meta data file
    if (status.isDir()) {
        Path candidatePath = new Path(path, SAVEPOINT_METADATA_FILE);
        if (fs.exists(candidatePath)) {
            path = candidatePath;
            LOG.info("Using savepoint file in {}", path);
        } else {
            throw new IOException("Cannot find meta data file in directory " + path + ". Please try to load the savepoint directly from the meta data file " + "instead of the directory.");
        }
    }
    // load the savepoint
    final Savepoint savepoint;
    try (DataInputStream dis = new DataInputViewStreamWrapper(fs.open(path))) {
        int magicNumber = dis.readInt();
        if (magicNumber == MAGIC_NUMBER) {
            int version = dis.readInt();
            SavepointSerializer<?> serializer = SavepointSerializers.getSerializer(version);
            savepoint = serializer.deserialize(dis, classLoader);
        } else {
            throw new RuntimeException("Unexpected magic number. This can have multiple reasons: " + "(1) You are trying to load a Flink 1.0 savepoint, which is not supported by this " + "version of Flink. (2) The file you were pointing to is not a savepoint at all. " + "(3) The savepoint file has been corrupted.");
        }
    }
    // construct the stream handle to the metadata file
    // we get the size best-effort
    long size = 0;
    try {
        size = fs.getFileStatus(path).getLen();
    } catch (Exception ignored) {
    // we don't know the size, but we don't want to fail the savepoint loading for that
    }
    StreamStateHandle metadataHandle = new FileStateHandle(path, size);
    return new Tuple2<>(savepoint, metadataHandle);
}

Example 4 with FileStatus

use of org.apache.flink.core.fs.FileStatus in project flink by apache.

the class FileCache method copy.

// ------------------------------------------------------------------------
//  Utilities
// ------------------------------------------------------------------------
public static void copy(Path sourcePath, Path targetPath, boolean executable) throws IOException {
    // TODO rewrite this to make it participate in the closable registry and the lifecycle of a task.
    // we unwrap the file system to get raw streams without safety net
    FileSystem sFS = FileSystem.getUnguardedFileSystem(sourcePath.toUri());
    FileSystem tFS = FileSystem.getUnguardedFileSystem(targetPath.toUri());
    if (!tFS.exists(targetPath)) {
        if (sFS.getFileStatus(sourcePath).isDir()) {
            tFS.mkdirs(targetPath);
            FileStatus[] contents = sFS.listStatus(sourcePath);
            for (FileStatus content : contents) {
                String distPath = content.getPath().toString();
                if (content.isDir()) {
                    if (distPath.endsWith("/")) {
                        distPath = distPath.substring(0, distPath.length() - 1);
                    }
                }
                String localPath = targetPath.toString() + distPath.substring(distPath.lastIndexOf("/"));
                copy(content.getPath(), new Path(localPath), executable);
            }
        } else {
            try (FSDataOutputStream lfsOutput = tFS.create(targetPath, false);
                FSDataInputStream fsInput = sFS.open(sourcePath)) {
                IOUtils.copyBytes(fsInput, lfsOutput);
                //noinspection ResultOfMethodCallIgnored
                new File(targetPath.toString()).setExecutable(executable);
            } catch (IOException ioe) {
                LOG.error("could not copy file to local file cache.", ioe);
            }
        }
    }
}

Example 5 with FileStatus

use of org.apache.flink.core.fs.FileStatus in project flink by apache.

the class AbstractFsCheckpointStorageAccess method resolveCheckpointPointer.

/**
 * Takes the given string (representing a pointer to a checkpoint) and resolves it to a file
 * status for the checkpoint's metadata file.
 *
 * @param checkpointPointer The pointer to resolve.
 * @return A state handle to checkpoint/savepoint's metadata.
 * @throws IOException Thrown, if the pointer cannot be resolved, the file system not accessed,
 *     or the pointer points to a location that does not seem to be a checkpoint/savepoint.
 */
@Internal
public static FsCompletedCheckpointStorageLocation resolveCheckpointPointer(String checkpointPointer) throws IOException {
    checkNotNull(checkpointPointer, "checkpointPointer");
    checkArgument(!checkpointPointer.isEmpty(), "empty checkpoint pointer");
    // check if the pointer is in fact a valid file path
    final Path path;
    try {
        path = new Path(checkpointPointer);
    } catch (Exception e) {
        throw new IOException("Checkpoint/savepoint path '" + checkpointPointer + "' is not a valid file URI. " + "Either the pointer path is invalid, or the checkpoint was created by a different state backend.");
    }
    // check if the file system can be accessed
    final FileSystem fs;
    try {
        fs = path.getFileSystem();
    } catch (IOException e) {
        throw new IOException("Cannot access file system for checkpoint/savepoint path '" + checkpointPointer + "'.", e);
    }
    final FileStatus status;
    try {
        status = fs.getFileStatus(path);
    } catch (FileNotFoundException e) {
        throw new FileNotFoundException("Cannot find checkpoint or savepoint " + "file/directory '" + checkpointPointer + "' on file system '" + fs.getUri().getScheme() + "'.");
    }
    // if we are here, the file / directory exists
    final Path checkpointDir;
    final FileStatus metadataFileStatus;
    // If this is a directory, we need to find the meta data file
    if (status.isDir()) {
        checkpointDir = status.getPath();
        final Path metadataFilePath = new Path(path, METADATA_FILE_NAME);
        try {
            metadataFileStatus = fs.getFileStatus(metadataFilePath);
        } catch (FileNotFoundException e) {
            throw new FileNotFoundException("Cannot find meta data file '" + METADATA_FILE_NAME + "' in directory '" + path + "'. Please try to load the checkpoint/savepoint " + "directly from the metadata file instead of the directory.");
        }
    } else {
        // this points to a file and we either do no name validation, or
        // the name is actually correct, so we can return the path
        metadataFileStatus = status;
        checkpointDir = status.getPath().getParent();
    }
    final FileStateHandle metaDataFileHandle = new FileStateHandle(metadataFileStatus.getPath(), metadataFileStatus.getLen());
    final String pointer = checkpointDir.makeQualified(fs).toString();
    return new FsCompletedCheckpointStorageLocation(fs, checkpointDir, metaDataFileHandle, pointer);
}