Examples with FilePath com.alibaba.alink.common.io.filesystem.FilePath used on opensource projects

Example 6 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class ResourcePluginFactory method getResourcePluginPath.

public static FilePath getResourcePluginPath(RegisterKey registerKey, RegisterKey... candidates) throws IOException {
    PluginDownloader pluginDownloader = new PluginDownloader();
    if (pluginDownloader.checkPluginExistRoughly(registerKey.getName(), registerKey.getVersion())) {
        return new FilePath(pluginDownloader.localResourcePluginPath(registerKey.getName(), registerKey.getVersion()), new LocalFileSystem());
    }
    for (RegisterKey candidate : candidates) {
        if (pluginDownloader.checkPluginExistRoughly(candidate.getName(), candidate.getVersion())) {
            return new FilePath(pluginDownloader.localResourcePluginPath(candidate.getName(), candidate.getVersion()), new LocalFileSystem());
        }
    }
    DistributeCache distributeCache = PluginDistributeCache.createDistributeCache(registerKey.getName(), registerKey.getVersion());
    distributeCache.distributeAsLocalFile();
    ResourcesPluginManager manager = PluginUtils.createResourcesPluginManagerFromRootFolder(PluginUtils.readPluginConf(distributeCache.context()));
    Iterator<ResourcesPluginDescriptor> iterator = manager.iterator(registerKey.getName(), registerKey.getVersion());
    if (iterator.hasNext()) {
        return iterator.next().getRootFolder();
    } else {
        throw new PluginNotExistException(String.format("Could not find the appropriate resource plugin. name: %s, version: %s", registerKey.getName(), registerKey.getVersion()));
    }
}

Example 7 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class ResourcesPluginDirectory method createPluginDescriptorForSubDirectory.

public ResourcesPluginDescriptor createPluginDescriptorForSubDirectory(String pluginName, String pluginVersion) {
    FilePath subDirectory = new FilePath(new Path(pluginsRootDir.getPath(), new Path(RESOURCE_FOLDER, String.format("%s-%s", pluginName, pluginVersion))), pluginsRootDir.getFileSystem());
    boolean folderExists;
    try {
        folderExists = subDirectory.getFileSystem().exists(subDirectory.getPath());
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
    if (folderExists) {
        return new ResourcesPluginDescriptor(subDirectory.getPath().getName(), pluginVersion, subDirectory);
    } else {
        return null;
    }
}

Example 8 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class PipelineSaveAndLoadTest method testLocalPredictor.

@Test
public void testLocalPredictor() throws Exception {
    VectorAssembler va = new VectorAssembler().setSelectedCols(Iris.getFeatureColNames()).setOutputCol("features");
    MultilayerPerceptronClassifier classifier = new MultilayerPerceptronClassifier().setVectorCol("features").setLabelCol(Iris.getLabelColName()).setLayers(new int[] { 4, 5, 3 }).setMaxIter(30).setPredictionCol("pred_label").setPredictionDetailCol("pred_detail").setReservedCols(Iris.getLabelColName());
    Pipeline pipeline = new Pipeline().add(va).add(classifier);
    PipelineModel model = pipeline.fit(data);
    FilePath filePath = new FilePath(folder.newFile().getAbsolutePath());
    model.save(filePath, true);
    BatchOperator.execute();
    LocalPredictor localPredictor = new LocalPredictor(filePath, new TableSchema(ArrayUtils.add(data.getColNames(), "features"), ArrayUtils.add(data.getColTypes(), VectorTypes.DENSE_VECTOR)));
    Row result = localPredictor.map(Row.of(5.1, 3.5, 1.4, 0.2, "Iris-setosanew", new DenseVector(new double[] { 5.1, 3.5, 1.4, 0.2 })));
    System.out.println(JsonConverter.toJson(result));
}

Example 9 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class LSTNetTrainBatchOpTest method testStreamMultiVar.

@Test
public void testStreamMultiVar() throws Exception {
    BatchOperator.setParallelism(1);
    final int numCols = 10;
    final String timeColName = "ts";
    final String vecColName = "vec";
    final String selectClause = "TO_TIMESTAMP(" + timeColName + ") as " + timeColName + ", " + vecColName;
    BatchOperator<?> source = new RandomTableSourceBatchOp().setNumRows(1000L).setNumCols(numCols);
    String[] selectedColNames = source.getColNames();
    AppendIdBatchOp appendIdBatchOp = new AppendIdBatchOp().setIdCol(timeColName).linkFrom(source);
    ColumnsToVectorBatchOp columnsToVectorBatchOp = new ColumnsToVectorBatchOp().setSelectedCols(selectedColNames).setVectorCol(vecColName).linkFrom(appendIdBatchOp);
    BatchOperator<?> timeBatchOp = new SelectBatchOp().setClause(selectClause).linkFrom(columnsToVectorBatchOp);
    LSTNetTrainBatchOp trainOp = new LSTNetTrainBatchOp().setVectorCol(vecColName).setTimeCol(timeColName).setWindow(24 * 7).setHorizon(12).setNumEpochs(1).linkFrom(timeBatchOp);
    StreamOperator<?> sourceStreamOp = new RandomTableSourceStreamOp().setNumCols(numCols).setMaxRows(1000L);
    ColumnsToVectorStreamOp columnsToVectorStreamOp = new ColumnsToVectorStreamOp().setSelectedCols(selectedColNames).setVectorCol(vecColName).linkFrom(sourceStreamOp);
    AppendIdStreamOp appendIdStreamOp = new AppendIdStreamOp().setIdCol(timeColName).linkFrom(columnsToVectorStreamOp);
    StreamOperator<?> timestampStreamOp = new SelectStreamOp().setClause(selectClause).linkFrom(appendIdStreamOp);
    OverCountWindowStreamOp overCountWindowStreamOp = new OverCountWindowStreamOp().setClause("MTABLE_AGG_PRECEDING(" + timeColName + ", " + vecColName + ") as col_agg").setTimeCol(timeColName).setPrecedingRows(24 * 7).linkFrom(timestampStreamOp);
    LSTNetPredictStreamOp predictStreamOp = new LSTNetPredictStreamOp(trainOp).setValueCol("col_agg").setPredictionCol("pred").setReservedCols(timeColName).linkFrom(overCountWindowStreamOp);
    FilePath tmpAkFile = new FilePath(new Path(folder.getRoot().getPath(), "lstnet_test_stream_multi_var_result.ak"));
    predictStreamOp.link(new AkSinkStreamOp().setOverwriteSink(true).setFilePath(tmpAkFile));
    StreamOperator.execute();
}

Example 10 with FilePath

use of com.alibaba.alink.common.io.filesystem.FilePath in project Alink by alibaba.

the class ReadAudioToTensorMapper method map.

@Override
protected void map(SlicedSelectedSample selection, SlicedResult result) throws Exception {
    Tuple2<Long, FloatTensor> res = AudioToFloatTensor.read(new FilePath(new Path(rootFolder.getPath(), (String) selection.get(0)), rootFolder.getFileSystem()), sampleRate, duration, offset);
    result.set(0, res.f1);
}