Examples with LocalFileSystem com.alibaba.alink.common.io.filesystem.LocalFileSystem used on opensource projects

Example 1 with LocalFileSystem

use of com.alibaba.alink.common.io.filesystem.LocalFileSystem in project Alink by alibaba.

the class Chap03 method c_1_2_2.

static void c_1_2_2() throws Exception {
    LocalFileSystem local = new LocalFileSystem();
    HadoopFileSystem hdfs = new HadoopFileSystem(HADOOP_VERSION, HDFS_URI);
    copy(hdfs.open(HDFS_URI + "user/yangxu/alink/data/temp/hello.txt"), local.create(LOCAL_DIR + "hello_1.txt", WriteMode.OVERWRITE));
    copy(local.open(LOCAL_DIR + "hello_1.txt"), hdfs.create(HDFS_URI + "user/yangxu/alink/data/temp/hello_2.txt", WriteMode.OVERWRITE));
    for (FileStatus status : hdfs.listStatus(HDFS_URI + "user/yangxu/alink/data/temp/")) {
        System.out.println(status.getPath().toUri() + " \t" + status.getLen() + " \t" + new Date(status.getModificationTime()));
    }
}

Example 2 with LocalFileSystem

use of com.alibaba.alink.common.io.filesystem.LocalFileSystem in project Alink by alibaba.

the class Chap03 method c_1_3_2.

static void c_1_3_2() throws Exception {
    LocalFileSystem local = new LocalFileSystem();
    OssFileSystem oss = new OssFileSystem(OSS_VERSION, OSS_END_POINT, OSS_BUCKET_NAME, OSS_ACCESS_ID, OSS_ACCESS_KEY);
    copy(oss.open(OSS_PREFIX_URI + "alink/data/temp/hello.txt"), local.create(LOCAL_DIR + "hello_1.txt", WriteMode.OVERWRITE));
    copy(local.open(LOCAL_DIR + "hello_1.txt"), oss.create(OSS_PREFIX_URI + "alink/data/temp/hello_2.txt", WriteMode.OVERWRITE));
    for (FileStatus status : oss.listStatus(new Path(OSS_PREFIX_URI + "alink/data/temp/"))) {
        System.out.println(status.getPath().toUri() + " \t" + status.getLen() + " \t" + new Date(status.getModificationTime()));
    }
}

Example 3 with LocalFileSystem

use of com.alibaba.alink.common.io.filesystem.LocalFileSystem in project Alink by alibaba.

the class HiveCatalog method fileExists.

public static boolean fileExists(FilePath folder, String file) throws IOException {
    // local
    if (folder.getFileSystem() instanceof LocalFileSystem) {
        return folder.getFileSystem().exists(new Path(folder.getPath(), file));
    }
    String scheme = folder.getPath().toUri().getScheme();
    if (scheme != null && (scheme.equalsIgnoreCase("http") || scheme.equalsIgnoreCase("https"))) {
        try (HttpFileSplitReader reader = new HttpFileSplitReader(folder.getPathStr() + "/" + file)) {
            long fileLen = reader.getFileLength();
            reader.open(null, 0, fileLen);
        } catch (FileNotFoundException exception) {
            return false;
        }
        return true;
    } else {
        return folder.getFileSystem().exists(new Path(folder.getPath(), file));
    }
}

Example 4 with LocalFileSystem

use of com.alibaba.alink.common.io.filesystem.LocalFileSystem in project Alink by alibaba.

the class ResourcePluginFactory method getResourcePluginPath.

public static FilePath getResourcePluginPath(RegisterKey registerKey, RegisterKey... candidates) throws IOException {
    PluginDownloader pluginDownloader = new PluginDownloader();
    if (pluginDownloader.checkPluginExistRoughly(registerKey.getName(), registerKey.getVersion())) {
        return new FilePath(pluginDownloader.localResourcePluginPath(registerKey.getName(), registerKey.getVersion()), new LocalFileSystem());
    }
    for (RegisterKey candidate : candidates) {
        if (pluginDownloader.checkPluginExistRoughly(candidate.getName(), candidate.getVersion())) {
            return new FilePath(pluginDownloader.localResourcePluginPath(candidate.getName(), candidate.getVersion()), new LocalFileSystem());
        }
    }
    DistributeCache distributeCache = PluginDistributeCache.createDistributeCache(registerKey.getName(), registerKey.getVersion());
    distributeCache.distributeAsLocalFile();
    ResourcesPluginManager manager = PluginUtils.createResourcesPluginManagerFromRootFolder(PluginUtils.readPluginConf(distributeCache.context()));
    Iterator<ResourcesPluginDescriptor> iterator = manager.iterator(registerKey.getName(), registerKey.getVersion());
    if (iterator.hasNext()) {
        return iterator.next().getRootFolder();
    } else {
        throw new PluginNotExistException(String.format("Could not find the appropriate resource plugin. name: %s, version: %s", registerKey.getName(), registerKey.getVersion()));
    }
}

Example 5 with LocalFileSystem

use of com.alibaba.alink.common.io.filesystem.LocalFileSystem in project Alink by alibaba.

the class HiveCatalog method downloadFolder.

public static String downloadFolder(FilePath folder, String... files) throws IOException {
    // local
    if (folder.getFileSystem() instanceof LocalFileSystem) {
        return folder.getPathStr();
    }
    File localConfDir = new File(System.getProperty("java.io.tmpdir"), FileUtils.getRandomFilename(""));
    String scheme = folder.getPath().toUri().getScheme();
    if (!localConfDir.mkdir()) {
        throw new RuntimeException("Could not create the dir " + localConfDir.getAbsolutePath());
    }
    if (scheme != null && (scheme.equalsIgnoreCase("http") || scheme.equalsIgnoreCase("https"))) {
        for (String path : files) {
            try (HttpFileSplitReader reader = new HttpFileSplitReader(folder.getPathStr() + "/" + path)) {
                long fileLen = reader.getFileLength();
                reader.open(null, 0, fileLen);
                int offset = 0;
                byte[] buffer = new byte[1024];
                try (FileOutputStream outputStream = new FileOutputStream(Paths.get(localConfDir.getPath(), path).toFile())) {
                    while (offset < fileLen) {
                        int len = reader.read(buffer, offset, 1024);
                        outputStream.write(buffer, offset, len);
                        offset += len;
                    }
                }
            } catch (FileNotFoundException exception) {
            // pass
            }
        }
    } else {
        for (String path : files) {
            // file system
            if (!folder.getFileSystem().exists(new Path(folder.getPath(), path))) {
                continue;
            }
            try (FSDataInputStream inputStream = folder.getFileSystem().open(new Path(folder.getPath(), path));
                FileOutputStream outputStream = new FileOutputStream(Paths.get(localConfDir.getPath(), path).toFile())) {
                IOUtils.copy(inputStream, outputStream);
            }
        }
    }
    return localConfDir.getAbsolutePath();
}