Examples with LogCopier org.apache.gobblin.util.logs.LogCopier used on opensource projects

Example 1 with LogCopier

use of org.apache.gobblin.util.logs.LogCopier in project incubator-gobblin by apache.

the class GobblinYarnLogSource method buildLogCopier.

/**
 * Build a {@link LogCopier} instance used to copy the logs out from this {@link GobblinYarnLogSource}.
 * TODO: This is duplicated to the org.apache.gobblin.yarn.GobblinYarnAppLauncher#buildLogCopier(com.typesafe.config.Config, org.apache.hadoop.fs.Path, org.apache.hadoop.fs.Path)
 *
 * @param config the {@link Config} use to create the {@link LogCopier}
 * @param containerId the {@link ContainerId} of the container the {@link LogCopier} runs in
 * @param destFs the destination {@link FileSystem}
 * @param appWorkDir the Gobblin Yarn application working directory on HDFS
 * @return a {@link LogCopier} instance
 * @throws IOException if it fails on any IO operation
 */
protected LogCopier buildLogCopier(Config config, String containerId, FileSystem destFs, Path appWorkDir) throws IOException {
    LogCopier.Builder builder = LogCopier.newBuilder().useDestFsSupplier(new FileSystemSupplier() {

        @Override
        public FileSystem getFileSystem() throws IOException {
            return buildFileSystem(config, false);
        }
    }).useSrcFsSupplier(new FileSystemSupplier() {

        @Override
        public FileSystem getFileSystem() throws IOException {
            return buildFileSystem(config, true);
        }
    }).readFrom(getLocalLogDirs()).writeTo(getHdfsLogDir(containerId, destFs, appWorkDir)).useCurrentLogFileName(Files.getNameWithoutExtension(System.getProperty(GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_LOG_FILE_NAME)));
    builder.acceptsLogFileExtensions(config.hasPath(GobblinYarnConfigurationKeys.LOG_FILE_EXTENSIONS) ? ImmutableSet.copyOf(Splitter.on(",").splitToList(config.getString(GobblinYarnConfigurationKeys.LOG_FILE_EXTENSIONS))) : ImmutableSet.of());
    return builder.build();
}

Example 2 with LogCopier

use of org.apache.gobblin.util.logs.LogCopier in project incubator-gobblin by apache.

the class GobblinYarnAppLauncher method buildLogCopier.

private LogCopier buildLogCopier(Config config, Path sinkLogDir, Path appWorkDir) throws IOException {
    FileSystem rawLocalFs = this.closer.register(new RawLocalFileSystem());
    rawLocalFs.initialize(URI.create(ConfigurationKeys.LOCAL_FS_URI), new Configuration());
    LogCopier.Builder builder = LogCopier.newBuilder().useSrcFileSystem(this.fs).useDestFileSystem(rawLocalFs).readFrom(getHdfsLogDir(appWorkDir)).writeTo(sinkLogDir).acceptsLogFileExtensions(ImmutableSet.of(ApplicationConstants.STDOUT, ApplicationConstants.STDERR));
    return builder.build();
}

Example 3 with LogCopier

use of org.apache.gobblin.util.logs.LogCopier in project incubator-gobblin by apache.

the class GobblinYarnTaskRunner method getServices.

@Override
public List<Service> getServices() {
    List<Service> services = new ArrayList<>();
    services.addAll(super.getServices());
    LogCopier logCopier = null;
    if (clusterConfig.hasPath(GobblinYarnConfigurationKeys.LOGS_SINK_ROOT_DIR_KEY)) {
        GobblinYarnLogSource gobblinYarnLogSource = new GobblinYarnLogSource();
        String containerLogDir = clusterConfig.getString(GobblinYarnConfigurationKeys.LOGS_SINK_ROOT_DIR_KEY);
        if (gobblinYarnLogSource.isLogSourcePresent()) {
            try {
                logCopier = gobblinYarnLogSource.buildLogCopier(this.clusterConfig, this.taskRunnerId, this.fs, new Path(containerLogDir, GobblinClusterUtils.getAppWorkDirPath(this.applicationName, this.applicationId)));
                services.add(logCopier);
            } catch (Exception e) {
                LOGGER.warn("Cannot add LogCopier service to the service manager due to", e);
            }
        }
    }
    if (UserGroupInformation.isSecurityEnabled()) {
        LOGGER.info("Adding YarnContainerSecurityManager since security is enabled");
        services.add(new YarnContainerSecurityManager(this.clusterConfig, this.fs, this.eventBus, logCopier));
    }
    return services;
}