Examples with FileSystemSupplier org.apache.gobblin.util.filesystem.FileSystemSupplier used on opensource projects

Example 1 with FileSystemSupplier

use of org.apache.gobblin.util.filesystem.FileSystemSupplier in project incubator-gobblin by apache.

the class GobblinYarnLogSource method buildLogCopier.

/**
 * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}.
 * TODO: This is duplicated to the org.apache.gobblin.yarn.GobblinYarnAppLauncher#buildLogCopier(com.typesafe.config.Config, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path)
 *
 * @param config the {@link Config} use to create the {@link LogCopier}
 * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in
 * @param destFs the destination {@link FileSystem}
 * @param appWorkDir the Gobblin Yarn application working directory on HDFS
 * @return a {@link LogCopier} instance
 * @throws IOException if it fails on any IO operation
 */
protected LogCopier buildLogCopier(Config config, String containerId, FileSystem destFs, Path appWorkDir) throws IOException {
    LogCopier.Builder builder = LogCopier.newBuilder().useDestFsSupplier(new FileSystemSupplier() {

        @Override
        public FileSystem getFileSystem() throws IOException {
            return buildFileSystem(config, false);
        }
    }).useSrcFsSupplier(new FileSystemSupplier() {

        @Override
        public FileSystem getFileSystem() throws IOException {
            return buildFileSystem(config, true);
        }
    }).readFrom(getLocalLogDirs()).writeTo(getHdfsLogDir(containerId, destFs, appWorkDir)).useCurrentLogFileName(Files.getNameWithoutExtension(System.getProperty(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME)));
    builder.acceptsLogFileExtensions(config.hasPath(GobblinYarnConfigurationKeys.LOG_FILE_EXTENSIONS) ? ImmutableSet.copyOf(Splitter.on(",").splitToList(config.getString(GobblinYarnConfigurationKeys.LOG_FILE_EXTENSIONS))) : ImmutableSet.of());
    return builder.build();
}