Examples with FilePath com.alibaba.alink.common.io.filesystem.FilePath used on opensource projects

Example 21 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class WriteTensorToImageMapper method map.

@Override
protected void map(SlicedSelectedSample selection, SlicedResult result) throws Exception {
    FloatTensorToImage.write(FloatTensor.of(TensorUtil.getTensor(selection.get(0))), new FilePath(new Path(rootPath.getPath(), (String) selection.get(1)), rootPath.getFileSystem()), imageType);
    result.set(0, selection.get(0));
    result.set(1, selection.get(1));
}

Example 22 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class FileModelStreamSink method open.

public void open(Timestamp modelId, int subId) throws IOException {
    BaseFileSystem<?> fileSystem = filePath.getFileSystem();
    Path confDirPath = new Path(filePath.getPath(), MODEL_CONF);
    Path fileInProgress = new Path(confDirPath, String.format("%s_%d", ModelStreamUtils.toStringPresentation(modelId), subId));
    collector = new AkStream(new FilePath(fileInProgress, fileSystem), new AkMeta(schemaStr)).getWriter().getCollector();
}

Example 23 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class LocalPredictorLoader method readPipelineModelRowsFromCsvFile.

@Deprecated
static Map<Long, List<Row>> readPipelineModelRowsFromCsvFile(String filePath, TableSchema schema) throws Exception {
    Map<Long, List<Row>> rows = new HashMap<>();
    Path path = new Path(filePath);
    FileSystem fs = FileSystem.get(path.toUri());
    FSDataInputStream stream = fs.open(path);
    BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
    CsvParser csvParser = new CsvParser(schema.getFieldTypes(), ",", '"');
    while (reader.ready()) {
        String line = reader.readLine();
        Tuple2<Boolean, Row> parsed = csvParser.parse(line);
        Preconditions.checkState(parsed.f0, "Fail to parse line: " + line);
        Long id = (Long) parsed.f1.getField(0);
        if (rows.containsKey(id)) {
            rows.get(id).add(parsed.f1);
        } else {
            List<Row> r = new ArrayList<>();
            r.add(parsed.f1);
            rows.put(id, r);
        }
    }
    reader.close();
    stream.close();
    return rows;
}

Example 24 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class BertResources method getBertResource.

static String getBertResource(ModelName modelName, ResourceType type) {
    String remotePath = BERT_RESOURCE_PATH_MAP.get(Pair.of(modelName, type));
    RegisterKey registerKey = getRegisterKey(modelName, type);
    FilePath pluginFilePath = null;
    try {
        pluginFilePath = ResourcePluginFactory.getResourcePluginPath(registerKey);
    } catch (IOException e) {
        // pass
        LOG.info("Could not find the plugin", e);
    }
    if (null != pluginFilePath) {
        String directoryName = PythonFileUtils.getCompressedFileName(remotePath);
        File file = new File(pluginFilePath.getPath().toString(), directoryName);
        Preconditions.checkArgument(file.exists() && file.isDirectory(), String.format("There should be a directory named %s in plugin directory %s, but cannot be found.", directoryName, pluginFilePath.getPath().toString()));
        return "file://" + file.getAbsolutePath();
    }
    // Use default PythonEnv path in PYTHON_ENV_MAP
    if (null == remotePath) {
        throw new RuntimeException(String.format("Default resource path for %s %s not specified.", modelName.name(), type.name()));
    }
    LOG.info("Use plugin resource: {}", remotePath);
    if (AlinkGlobalConfiguration.isPrintProcessInfo()) {
        System.out.println(String.format("Use plugin resource:%s", remotePath));
    }
    return remotePath;
}

Example 25 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class HiveCatalog method downloadFolder.

public static String downloadFolder(FilePath folder, String... files) throws IOException {
    // local
    if (folder.getFileSystem() instanceof LocalFileSystem) {
        return folder.getPathStr();
    }
    File localConfDir = new File(System.getProperty("java.io.tmpdir"), FileUtils.getRandomFilename(""));
    String scheme = folder.getPath().toUri().getScheme();
    if (!localConfDir.mkdir()) {
        throw new RuntimeException("Could not create the dir " + localConfDir.getAbsolutePath());
    }
    if (scheme != null && (scheme.equalsIgnoreCase("http") || scheme.equalsIgnoreCase("https"))) {
        for (String path : files) {
            try (HttpFileSplitReader reader = new HttpFileSplitReader(folder.getPathStr() + "/" + path)) {
                long fileLen = reader.getFileLength();
                reader.open(null, 0, fileLen);
                int offset = 0;
                byte[] buffer = new byte[1024];
                try (FileOutputStream outputStream = new FileOutputStream(Paths.get(localConfDir.getPath(), path).toFile())) {
                    while (offset < fileLen) {
                        int len = reader.read(buffer, offset, 1024);
                        outputStream.write(buffer, offset, len);
                        offset += len;
                    }
                }
            } catch (FileNotFoundException exception) {
            // pass
            }
        }
    } else {
        for (String path : files) {
            // file system
            if (!folder.getFileSystem().exists(new Path(folder.getPath(), path))) {
                continue;
            }
            try (FSDataInputStream inputStream = folder.getFileSystem().open(new Path(folder.getPath(), path));
                FileOutputStream outputStream = new FileOutputStream(Paths.get(localConfDir.getPath(), path).toFile())) {
                IOUtils.copy(inputStream, outputStream);
            }
        }
    }
    return localConfDir.getAbsolutePath();
}