Search in sources :

Example 1 with DistributePluginException

use of com.alibaba.alink.common.exceptions.DistributePluginException in project Alink by alibaba.

the class PluginDownloader method loadPluginConfig.

private static String loadPluginConfig(String configFileName, FilePath remotePath, String localPath, boolean force) throws IOException {
    final FilePath remoteFilePath = new FilePath(new Path(remotePath.getPath(), configFileName), remotePath.getFileSystem());
    downloadFileLocked(localPath, configFileName, (rawPath, path) -> {
        File rawFile = new File(rawPath.getPath());
        if (force || !rawFile.exists()) {
            download(remoteFilePath, path.getPath());
            if (force) {
                rawFile.delete();
            }
            if (!new File(path.getPath()).renameTo(new File(rawPath.getPath()))) {
                throw new DistributePluginException(String.format("Commit file: %s fail.", path.getPath()));
            }
        }
    });
    return Files.toString(new File(new Path(localPath, configFileName).getPath()), StandardCharsets.UTF_8);
}
Also used : FilePath(com.alibaba.alink.common.io.filesystem.FilePath) Path(org.apache.flink.core.fs.Path) FilePath(com.alibaba.alink.common.io.filesystem.FilePath) DistributePluginException(com.alibaba.alink.common.exceptions.DistributePluginException) File(java.io.File)

Example 2 with DistributePluginException

use of com.alibaba.alink.common.exceptions.DistributePluginException in project Alink by alibaba.

the class PluginDownloader method downloadPluginSafely.

public void downloadPluginSafely(String pluginName, String pluginVersion) throws IOException {
    loadPluginConfig();
    String pluginFolder = pluginName + "-" + pluginVersion;
    if (jarsPluginConfigs.containsKey(pluginName)) {
        final List<String> jars = getListOfJars(pluginName, pluginVersion);
        final FilePath remotePath = new FilePath(new Path(getRemoteFlinkRoot().getPath(), pluginFolder), getRemoteFlinkRoot().getFileSystem());
        downloadFileLocked(getLocalFlinkRoot(), pluginFolder, (rawPath, path) -> {
            if (new File(rawPath.getPath()).exists()) {
                return;
            }
            File f = new File(path.getPath());
            if (f.exists()) {
                // overwrite tmp file.
                LOG.info("Tmp file {} exists. Delete first.", f);
                try {
                    FileUtils.forceDelete(f);
                } catch (IOException e) {
                    LOG.warn("Delete tmp file {} returns false.", f, e);
                }
            }
            for (String jar : jars) {
                download(new FilePath(new Path(remotePath.getPath(), jar), remotePath.getFileSystem()), new Path(path, jar).getPath());
            }
            if (!new File(path.getPath()).renameTo(new File(rawPath.getPath()))) {
                throw new DistributePluginException(String.format("Commit file: %s fail.", path.getPath()));
            }
        });
    } else if (resourcePluginConfigs.containsKey(pluginName)) {
        final List<String> resources = getListOfResource(pluginName, pluginVersion);
        final FilePath remotePath = new FilePath(new Path(getRemoteResourceRoot().getPath(), pluginFolder), getRemoteResourceRoot().getFileSystem());
        downloadFileLocked(getLocalResourceRoot(), pluginFolder, (rawPath, path) -> {
            if (new File(rawPath.getPath()).exists()) {
                return;
            }
            File f = new File(path.getPath());
            if (f.exists()) {
                // overwrite tmp file.
                LOG.info("Tmp file {} exists. Delete first.", f);
                try {
                    FileUtils.forceDelete(f);
                } catch (IOException e) {
                    LOG.warn("Delete tmp file {} returns false.", f, e);
                }
            }
            for (String resource : resources) {
                download(new FilePath(new Path(remotePath.getPath(), resource), remotePath.getFileSystem()), new Path(path, resource).getPath());
            }
            if (!new File(path.getPath()).renameTo(new File(rawPath.getPath()))) {
                throw new DistributePluginException(String.format("Commit file: %s fail.", path.getPath()));
            }
        });
    } else {
        throw new DistributePluginException("plugin [" + pluginName + "] not found!");
    }
}
Also used : FilePath(com.alibaba.alink.common.io.filesystem.FilePath) Path(org.apache.flink.core.fs.Path) FilePath(com.alibaba.alink.common.io.filesystem.FilePath) LoggerFactory(org.slf4j.LoggerFactory) Files(org.apache.flink.shaded.guava18.com.google.common.io.Files) JsonConverter(com.alibaba.alink.common.utils.JsonConverter) FileLock(java.nio.channels.FileLock) ArrayList(java.util.ArrayList) Path(org.apache.flink.core.fs.Path) Map(java.util.Map) Logger(org.slf4j.Logger) ZipFileUtil(com.alibaba.alink.common.dl.utils.ZipFileUtil) FileOutputStream(java.io.FileOutputStream) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) AlinkGlobalConfiguration(com.alibaba.alink.common.AlinkGlobalConfiguration) TarFileUtil(com.alibaba.alink.common.pyrunner.TarFileUtil) File(java.io.File) StandardCharsets(java.nio.charset.StandardCharsets) BaseFileSystem(com.alibaba.alink.common.io.filesystem.BaseFileSystem) VisibleForTesting(org.apache.flink.annotation.VisibleForTesting) TypeReference(org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.type.TypeReference) List(java.util.List) FilePath(com.alibaba.alink.common.io.filesystem.FilePath) DistributePluginException(com.alibaba.alink.common.exceptions.DistributePluginException) FileChannel(java.nio.channels.FileChannel) Collections(java.util.Collections) InputStream(java.io.InputStream) DistributePluginException(com.alibaba.alink.common.exceptions.DistributePluginException) ArrayList(java.util.ArrayList) List(java.util.List) IOException(java.io.IOException) File(java.io.File)

Aggregations

DistributePluginException (com.alibaba.alink.common.exceptions.DistributePluginException)2 FilePath (com.alibaba.alink.common.io.filesystem.FilePath)2 File (java.io.File)2 Path (org.apache.flink.core.fs.Path)2 AlinkGlobalConfiguration (com.alibaba.alink.common.AlinkGlobalConfiguration)1 ZipFileUtil (com.alibaba.alink.common.dl.utils.ZipFileUtil)1 BaseFileSystem (com.alibaba.alink.common.io.filesystem.BaseFileSystem)1 TarFileUtil (com.alibaba.alink.common.pyrunner.TarFileUtil)1 JsonConverter (com.alibaba.alink.common.utils.JsonConverter)1 FileOutputStream (java.io.FileOutputStream)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 FileChannel (java.nio.channels.FileChannel)1 FileLock (java.nio.channels.FileLock)1 StandardCharsets (java.nio.charset.StandardCharsets)1 ArrayList (java.util.ArrayList)1 Collections (java.util.Collections)1 List (java.util.List)1 Map (java.util.Map)1 FileUtils (org.apache.commons.io.FileUtils)1