Examples with FileSystem org.apache.flink.core.fs.FileSystem used on opensource projects

Example 56 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class ContinuousFileMonitoringFunction method run.

@Override
public void run(SourceFunction.SourceContext<TimestampedFileInputSplit> context) throws Exception {
    Path p = new Path(path);
    FileSystem fileSystem = FileSystem.get(p.toUri());
    if (!fileSystem.exists(p)) {
        throw new FileNotFoundException("The provided file path " + path + " does not exist.");
    }
    checkpointLock = context.getCheckpointLock();
    switch(watchType) {
        case PROCESS_CONTINUOUSLY:
            while (isRunning) {
                synchronized (checkpointLock) {
                    monitorDirAndForwardSplits(fileSystem, context);
                }
                Thread.sleep(interval);
            }
            break;
        case PROCESS_ONCE:
            synchronized (checkpointLock) {
                if (globalModificationTime == Long.MIN_VALUE) {
                    monitorDirAndForwardSplits(fileSystem, context);
                    globalModificationTime = Long.MAX_VALUE;
                }
                isRunning = false;
            }
            break;
        default:
            isRunning = false;
            throw new RuntimeException("Unknown WatchType" + watchType);
    }
}

Example 57 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class AbstractFsStateSnapshot method deserialize.

@Override
@SuppressWarnings("unchecked")
public StateTable<K, N, SV> deserialize(String stateName, HeapKeyedStateBackend<K> stateBackend) throws IOException {
    final FileSystem fs = getFilePath().getFileSystem();
    try (FSDataInputStream inStream = fs.open(getFilePath())) {
        final DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(inStream);
        AbstractMigrationRestoreStrategy<K, N, SV> restoreStrategy = new AbstractMigrationRestoreStrategy<K, N, SV>(keySerializer, namespaceSerializer, stateSerializer) {

            @Override
            protected DataInputView openDataInputView() throws IOException {
                return inView;
            }
        };
        return restoreStrategy.deserialize(stateName, stateBackend);
    }
}

Example 58 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class BlobClient method uploadJarFiles.

/**
	 * Uploads the JAR files to a {@link BlobServer} at the given address.
	 *
	 * @param serverAddress Server address of the {@link BlobServer}
	 * @param clientConfig Any additional configuration for the blob client
	 * @param jars List of JAR files to upload
	 * @throws IOException Thrown if the upload fails
	 */
public static List<BlobKey> uploadJarFiles(InetSocketAddress serverAddress, Configuration clientConfig, List<Path> jars) throws IOException {
    if (jars.isEmpty()) {
        return Collections.emptyList();
    } else {
        List<BlobKey> blobKeys = new ArrayList<>();
        try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) {
            for (final Path jar : jars) {
                final FileSystem fs = jar.getFileSystem();
                FSDataInputStream is = null;
                try {
                    is = fs.open(jar);
                    final BlobKey key = blobClient.put(is);
                    blobKeys.add(key);
                } finally {
                    if (is != null) {
                        is.close();
                    }
                }
            }
        }
        return blobKeys;
    }
}

Example 59 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class SavepointStore method storeSavepointToHandle.

/**
	 * Stores the savepoint metadata file to a state handle.
	 *
	 * @param directory Target directory to store savepoint in
	 * @param savepoint Savepoint to be stored
	 *
	 * @return State handle to the checkpoint metadata
	 * @throws IOException Failures during store are forwarded
	 */
static <T extends Savepoint> FileStateHandle storeSavepointToHandle(String directory, String filename, T savepoint) throws IOException {
    checkNotNull(directory, "Target directory");
    checkNotNull(savepoint, "Savepoint");
    final Path basePath = new Path(directory);
    final Path metadataFilePath = new Path(basePath, filename);
    final FileSystem fs = FileSystem.get(basePath.toUri());
    boolean success = false;
    try (FSDataOutputStream fdos = fs.create(metadataFilePath, WriteMode.NO_OVERWRITE);
        DataOutputStream dos = new DataOutputStream(fdos)) {
        // Write header
        dos.writeInt(MAGIC_NUMBER);
        dos.writeInt(savepoint.getVersion());
        // Write savepoint
        SavepointSerializer<T> serializer = SavepointSerializers.getSerializer(savepoint);
        serializer.serialize(savepoint, dos);
        // construct result handle
        FileStateHandle handle = new FileStateHandle(metadataFilePath, dos.size());
        // all good!
        success = true;
        return handle;
    } finally {
        if (!success && fs.exists(metadataFilePath)) {
            if (!fs.delete(metadataFilePath, true)) {
                LOG.warn("Failed to delete file {} after failed metadata write.", metadataFilePath);
            }
        }
    }
}

Example 60 with FileSystem

use of org.apache.flink.core.fs.FileSystem in project flink by apache.

the class Plan method registerCachedFile.

/**
	 *  register cache files in program level
	 * @param entry contains all relevant information
	 * @param name user defined name of that file
	 * @throws java.io.IOException
	 */
public void registerCachedFile(String name, DistributedCacheEntry entry) throws IOException {
    if (!this.cacheFile.containsKey(name)) {
        try {
            URI u = new URI(entry.filePath);
            if (!u.getPath().startsWith("/")) {
                u = new File(entry.filePath).toURI();
            }
            FileSystem fs = FileSystem.get(u);
            if (fs.exists(new Path(u.getPath()))) {
                this.cacheFile.put(name, new DistributedCacheEntry(u.toString(), entry.isExecutable));
            } else {
                throw new IOException("File " + u.toString() + " doesn't exist.");
            }
        } catch (URISyntaxException ex) {
            throw new IOException("Invalid path: " + entry.filePath, ex);
        }
    } else {
        throw new IOException("cache file " + name + "already exists!");
    }
}