Examples with FilePath com.alibaba.alink.common.io.filesystem.FilePath used on opensource projects

Example 1 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class HiveClassLoaderFactory method doAs.

@Override
public <T> T doAs(PrivilegedExceptionAction<T> action) throws Exception {
    ClassLoader classLoader = create();
    try (TemporaryClassLoaderContext context = TemporaryClassLoaderContext.of(classLoader)) {
        if (actionContext.get(HiveCatalogParams.KERBEROS_PRINCIPAL) == null || actionContext.get(HiveCatalogParams.KERBEROS_KEYTAB) == null) {
            return action.run();
        }
        if (internal == null) {
            String kerberosPrincipal = actionContext.get(HiveCatalogParams.KERBEROS_PRINCIPAL);
            FilePath filePath = FilePath.deserialize(actionContext.get(HiveCatalogParams.KERBEROS_KEYTAB));
            String kerberosKeytab;
            kerberosKeytab = new Path(HiveCatalog.downloadFolder(new FilePath(filePath.getPath().getParent(), filePath.getFileSystem()), filePath.getPath().getName()), filePath.getPath().getName()).toString();
            internal = createDoAs(kerberosPrincipal, kerberosKeytab, create());
        }
        return (T) internal.map((PrivilegedExceptionAction<Object>) action);
    }
}

Example 2 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class Chap03 method c_2_3_1.

static void c_2_3_1() throws Exception {
    HadoopFileSystem hdfs = new HadoopFileSystem(HADOOP_VERSION, HDFS_URI);
    OssFileSystem oss = new OssFileSystem(OSS_VERSION, OSS_END_POINT, OSS_BUCKET_NAME, OSS_ACCESS_ID, OSS_ACCESS_KEY);
    FilePath[] filePaths = new FilePath[] { new FilePath(LOCAL_DIR + "iris.ak"), new FilePath(HDFS_URI + "user/yangxu/alink/data/temp/iris.ak", hdfs), new FilePath(OSS_PREFIX_URI + "alink/data/temp/iris.ak", oss) };
    for (FilePath filePath : filePaths) {
        new CsvSourceBatchOp().setFilePath(IRIS_HTTP_URL).setSchemaStr(IRIS_SCHEMA_STR).link(new AkSinkBatchOp().setFilePath(filePath).setOverwriteSink(true));
        BatchOperator.execute();
        System.out.println(new AkSourceBatchOp().setFilePath(filePath).count());
    }
    for (FilePath filePath : filePaths) {
        new CsvSourceStreamOp().setFilePath(IRIS_HTTP_URL).setSchemaStr(IRIS_SCHEMA_STR).link(new AkSinkStreamOp().setFilePath(filePath).setOverwriteSink(true));
        StreamOperator.execute();
        new AkSourceStreamOp().setFilePath(filePath).filter("sepal_length < 4.5").print();
        StreamOperator.execute();
    }
}

Example 3 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class HiveCatalog method fileExists.

public static boolean fileExists(FilePath folder, String file) throws IOException {
    // local
    if (folder.getFileSystem() instanceof LocalFileSystem) {
        return folder.getFileSystem().exists(new Path(folder.getPath(), file));
    }
    String scheme = folder.getPath().toUri().getScheme();
    if (scheme != null && (scheme.equalsIgnoreCase("http") || scheme.equalsIgnoreCase("https"))) {
        try (HttpFileSplitReader reader = new HttpFileSplitReader(folder.getPathStr() + "/" + file)) {
            long fileLen = reader.getFileLength();
            reader.open(null, 0, fileLen);
        } catch (FileNotFoundException exception) {
            return false;
        }
        return true;
    } else {
        return folder.getFileSystem().exists(new Path(folder.getPath(), file));
    }
}

Example 4 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class PluginDownloader method downloadPlugin.

public void downloadPlugin(String pluginName, String pluginVersion) throws IOException {
    loadPluginConfig();
    String pluginFolder = pluginName + "-" + pluginVersion;
    if (jarsPluginConfigs.containsKey(pluginName)) {
        List<String> jars = getListOfJars(pluginName, pluginVersion);
        FilePath remotePath = new FilePath(new Path(getRemoteFlinkRoot().getPath(), pluginFolder), getRemoteFlinkRoot().getFileSystem());
        Path localPath = new Path(getLocalFlinkRoot(), pluginFolder);
        for (String jar : jars) {
            download(new FilePath(new Path(remotePath.getPath(), jar), remotePath.getFileSystem()), new Path(localPath, jar).getPath());
        }
    } else if (resourcePluginConfigs.containsKey(pluginName)) {
        List<String> resources = getListOfResource(pluginName, pluginVersion);
        FilePath remotePath = new FilePath(new Path(getRemoteResourceRoot().getPath(), pluginFolder), getRemoteResourceRoot().getFileSystem());
        Path localPath = new Path(getLocalResourceRoot(), pluginFolder);
        for (String resource : resources) {
            download(new FilePath(new Path(remotePath.getPath(), resource), remotePath.getFileSystem()), new Path(localPath, resource).getPath());
        }
    } else {
        throw new IllegalArgumentException("plugin [" + pluginName + "] not found!");
    }
}

Example 5 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class PluginDownloader method loadPluginConfig.

private static String loadPluginConfig(String configFileName, FilePath remotePath, String localPath, boolean force) throws IOException {
    final FilePath remoteFilePath = new FilePath(new Path(remotePath.getPath(), configFileName), remotePath.getFileSystem());
    downloadFileLocked(localPath, configFileName, (rawPath, path) -> {
        File rawFile = new File(rawPath.getPath());
        if (force || !rawFile.exists()) {
            download(remoteFilePath, path.getPath());
            if (force) {
                rawFile.delete();
            }
            if (!new File(path.getPath()).renameTo(new File(rawPath.getPath()))) {
                throw new DistributePluginException(String.format("Commit file: %s fail.", path.getPath()));
            }
        }
    });
    return Files.toString(new File(new Path(localPath, configFileName).getPath()), StandardCharsets.UTF_8);
}