Examples with TemporaryClassLoaderContext com.alibaba.alink.common.io.plugin.TemporaryClassLoaderContext used on opensource projects

Example 1 with TemporaryClassLoaderContext

use of com.alibaba.alink.common.io.plugin.TemporaryClassLoaderContext in project Alink by alibaba.

the class HiveClassLoaderFactory method doAs.

@Override
public <T> T doAs(PrivilegedExceptionAction<T> action) throws Exception {
    ClassLoader classLoader = create();
    try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(classLoader)) {
        if (actionContext.get(HiveCatalogParams.KERBEROS_PRINCIPAL) == null || actionContext.get(HiveCatalogParams.KERBEROS_KEYTAB) == null) {
            return action.run();
        }
        if (internal == null) {
            String kerberosPrincipal = actionContext.get(HiveCatalogParams.KERBEROS_PRINCIPAL);
            FilePath filePath = FilePath.deserialize(actionContext.get(HiveCatalogParams.KERBEROS_KEYTAB));
            String kerberosKeytab;
            kerberosKeytab = new Path(HiveCatalog.downloadFolder(new FilePath(filePath.getPath().getParent(), filePath.getFileSystem()), filePath.getPath().getName()), filePath.getPath().getName()).toString();
            internal = createDoAs(kerberosPrincipal, kerberosKeytab, create());
        }
        return (T) internal.map((PrivilegedExceptionAction<Object>) action);
    }
}

Example 2 with TemporaryClassLoaderContext

use of com.alibaba.alink.common.io.plugin.TemporaryClassLoaderContext in project Alink by alibaba.

the class S3FileSystem method load.

@Override
protected FileSystem load(Path path) {
    if (loaded != null) {
        return loaded;
    }
    final Configuration conf = new Configuration();
    conf.setString("s3.endpoint", getParams().get(S3FileSystemParams.END_POINT));
    if (getParams().get(S3FileSystemParams.ACCESS_KEY) != null && getParams().get(S3FileSystemParams.SECRET_KEY) != null) {
        conf.setString("s3.access-key", getParams().get(S3FileSystemParams.ACCESS_KEY));
        conf.setString("s3.secret-key", getParams().get(S3FileSystemParams.SECRET_KEY));
    }
    conf.setString("s3.path.style.access", getParams().get(S3FileSystemParams.PATH_STYLE_ACCESS).toString());
    List<FileSystemFactory> factories = createFactory();
    Path localPath = path;
    if (getParams().get(S3FileSystemParams.FS_URI) != null) {
        localPath = new Path(getParams().get(S3FileSystemParams.FS_URI));
    }
    for (FileSystemFactory factory : factories) {
        factory.configure(conf);
        if (localPath.toUri().getScheme().equals(factory.getScheme())) {
            try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(factory.getClassLoader())) {
                loaded = factory.create(localPath.toUri());
                return loaded;
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }
    }
    throw new IllegalStateException("Could not find the file system of " + localPath.toString());
}

Example 3 with TemporaryClassLoaderContext

use of com.alibaba.alink.common.io.plugin.TemporaryClassLoaderContext in project Alink by alibaba.

the class FileOutputFormat method open.

@Override
public void open(int taskNumber, int numTasks) throws IOException {
    if (taskNumber < 0 || numTasks < 1) {
        throw new IllegalArgumentException("TaskNumber: " + taskNumber + ", numTasks: " + numTasks);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Opening stream for output (" + (taskNumber + 1) + "/" + numTasks + "). WriteMode=" + writeMode + ", OutputDirectoryMode=" + outputDirectoryMode);
    }
    Path p = this.outputFilePath;
    if (p == null) {
        throw new IOException("The file path is null.");
    }
    final FileSystem fs = FileSystemUtils.getFlinkFileSystem(this.fs, p.toString());
    // if this is a local file system, we need to initialize the local output directory here
    if (!fs.isDistributedFS()) {
        if (numTasks == 1 && outputDirectoryMode == OutputDirectoryMode.PARONLY) {
            // mode
            if (!fs.initOutPathLocalFS(p, writeMode, false)) {
                // output preparation failed! Cancel task.
                throw new IOException("Output path '" + p.toString() + "' could not be initialized. Canceling task...");
            }
        } else {
            if (!fs.initOutPathLocalFS(p, writeMode, true)) {
                // output preparation failed! Cancel task.
                throw new IOException("Output directory '" + p.toString() + "' could not be created. Canceling task...");
            }
        }
    }
    // Suffix the path with the parallel instance index, if needed
    this.actualFilePath = (numTasks > 1 || outputDirectoryMode == OutputDirectoryMode.ALWAYS) ? p.suffix("/" + getDirectoryFileName(taskNumber)) : p;
    // create output file
    try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(fs.getClass().getClassLoader())) {
        this.stream = fs.create(this.actualFilePath, writeMode);
    }
    // at this point, the file creation must have succeeded, or an exception has been thrown
    this.fileCreated = true;
}

Example 4 with TemporaryClassLoaderContext

use of com.alibaba.alink.common.io.plugin.TemporaryClassLoaderContext in project Alink by alibaba.

the class HadoopFileSystem method load.

@Override
protected FileSystem load(Path path) {
    if (loaded != null) {
        return loaded;
    }
    final Configuration configuration = new Configuration();
    try {
        if (getParams().contains(HadoopFileSystemParams.CONFIGURATION)) {
            final String hdfsSiteFileName = "hdfs-site.xml";
            configuration.setString(ConfigConstants.HDFS_SITE_CONFIG, new Path(write(getParams().get(HadoopFileSystemParams.CONFIGURATION), hdfsSiteFileName), hdfsSiteFileName).toString());
        } else if (getParams().contains(HadoopFileSystemParams.CONFIGURATION_FILE_PATH)) {
            FilePath configurePath = FilePath.deserialize(getParams().get(HadoopFileSystemParams.CONFIGURATION_FILE_PATH));
            configuration.setString(ConfigConstants.HDFS_SITE_CONFIG, new Path(FilePath.download(new FilePath(configurePath.getPath().getParent(), configurePath.getFileSystem()), configurePath.getPath().getName()), configurePath.getPath().getName()).toString());
        }
        FileSystemFactory factory = createFactory();
        factory.configure(configuration);
        if (getParams().get(HadoopFileSystemParams.FS_URI) != null) {
            try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(factory.getClassLoader())) {
                loaded = factory.create(new Path(getParams().get(HadoopFileSystemParams.FS_URI)).toUri());
            }
            return loaded;
        } else if (path != null) {
            try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(factory.getClassLoader())) {
                loaded = factory.create(path.toUri());
            }
            return loaded;
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    throw new RuntimeException("Could not create the hadoop file system. Both the fsUri the filePath are null.");
}

Example 5 with TemporaryClassLoaderContext

use of com.alibaba.alink.common.io.plugin.TemporaryClassLoaderContext in project Alink by alibaba.

the class OssFileSystem method load.

@Override
protected FileSystem load(Path path) {
    if (loaded != null) {
        return loaded;
    }
    final Configuration conf = new Configuration();
    conf.setString("fs.oss.endpoint", getParams().get(OssFileSystemParams.END_POINT));
    if (getParams().get(OssFileSystemParams.ACCESS_ID) != null && getParams().get(OssFileSystemParams.ACCESS_KEY) != null) {
        conf.setString("fs.oss.accessKeyId", getParams().get(OssFileSystemParams.ACCESS_ID));
        conf.setString("fs.oss.accessKeySecret", getParams().get(OssFileSystemParams.ACCESS_KEY));
        if (getParams().get(OssFileSystemParams.SECURITY_TOKEN) != null) {
            conf.setString("fs.oss.securityToken", getParams().get(OssFileSystemParams.SECURITY_TOKEN));
        }
    }
    FileSystemFactory factory = createFactory();
    factory.configure(conf);
    try {
        if (getParams().get(OssFileSystemParams.FS_URI) != null) {
            try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(factory.getClassLoader())) {
                loaded = factory.create(new Path(getParams().get(OssFileSystemParams.FS_URI)).toUri());
            }
            return loaded;
        } else if (path != null) {
            try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(factory.getClassLoader())) {
                loaded = factory.create(path.toUri());
            }
            return loaded;
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    throw new RuntimeException("Could not create the oss file system. Both the bucket the filePath are null.");
}