Examples with MatrixFilesMeta com.tencent.angel.model.output.format.MatrixFilesMeta used on opensource projects

Example 11 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class ModelLoader method loadToFloatMaps.

/**
 * Load dense double model to int->float maps
 *
 * @param modelDir model save directory path
 * @return model data
 */
public static Int2FloatOpenHashMap[] loadToFloatMaps(String modelDir, Configuration conf) throws IOException {
    // Load model meta
    MatrixFilesMeta meta = getMeta(modelDir, conf);
    RowType rowType = RowType.valueOf(meta.getRowType());
    // Check row type
    if (rowType != RowType.T_FLOAT_SPARSE && rowType != rowType.T_FLOAT_SPARSE_COMPONENT) {
        throw new IOException("model row type is not sparse float, you should check it");
    }
    // Load model
    SparseFloatModel model = new SparseFloatModel(meta.getRow(), meta.getCol());
    loadModel(modelDir, model, meta, conf);
    return model.getModel();
}

Example 12 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class AMModelSaver method combineMatrix.

/**
 * Combine all output files of a model to a combine directory
 *
 * @param matrixContext matrix save context
 * @param errorLogs error logs
 */
private void combineMatrix(ModelSaveContext saveContext, MatrixSaveContext matrixContext, Vector<String> errorLogs, Path tmpCombinePath, FileSystem fs) {
    LOG.info("start commit matrix " + matrixContext.getMatrixName());
    // Init matrix files meta
    int matrixId = context.getMatrixMetaManager().getMatrix(matrixContext.getMatrixName()).getId();
    List<ParameterServerId> psIds = new ArrayList<>(context.getMatrixMetaManager().getMasterPsIds(matrixId));
    MatrixMeta meta = context.getMatrixMetaManager().getMatrix(matrixId);
    Map<String, String> kvMap = meta.getAttributes();
    MatrixFilesMeta filesMeta = new MatrixFilesMeta(matrixId, meta.getName(), matrixContext.getFormatClassName(), meta.getRowType().getNumber(), meta.getRowNum(), meta.getColNum(), meta.getBlockRowNum(), meta.getBlockColNum(), kvMap);
    filesMeta.setFeatureIndexStart(meta.getIndexStart());
    filesMeta.setFeatureIndexEnd(meta.getIndexEnd());
    try {
        // Move output files
        Path srcPath = new Path(saveContext.getTmpSavePath(), ModelFilesConstent.resultDirName);
        Path destPath = new Path(tmpCombinePath, meta.getName());
        PSModelCombineOp partCombineOp = new PSModelCombineOp(srcPath, destPath, psIds, errorLogs, filesMeta, 0, psIds.size(), fs);
        fileOpExecutor.execute(partCombineOp);
        partCombineOp.join();
        // Write the meta file
        long startTs = System.currentTimeMillis();
        Path metaFile = new Path(destPath, ModelFilesConstent.modelMetaFileName);
        Path tmpMetaFile = HdfsUtil.toTmpPath(metaFile);
        FSDataOutputStream metaOut = fs.create(tmpMetaFile);
        filesMeta.write(metaOut);
        metaOut.flush();
        metaOut.close();
        HdfsUtil.rename(tmpMetaFile, metaFile, fs);
        LOG.info("commit meta file use time=" + (System.currentTimeMillis() - startTs));
    } catch (Throwable x) {
        errorLogs.add("move output files for matrix " + meta.getName() + " failed, error msg = " + x.getMessage());
        LOG.error("move output files for matrix " + meta.getName() + " failed.", x);
    }
}

Example 13 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class AMModelLoader method split.

private Map<ParameterServerId, PSMatrixLoadContext> split(MatrixLoadContext matrixLoadContext, ModelLoadContext modelLoadContext) throws IOException {
    Path matrixPath = new Path(modelLoadContext.getLoadPath(), matrixLoadContext.getMatrixName());
    Path metaFilePath = new Path(matrixPath, ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta matrixFilesMeta = new MatrixFilesMeta();
    FileSystem fs = metaFilePath.getFileSystem(context.getConf());
    if (fs.exists(metaFilePath)) {
        FSDataInputStream input = fs.open(metaFilePath);
        try {
            matrixFilesMeta.read(input);
        } catch (Throwable e) {
            throw new IOException("Read matrix meta failed ", e);
        } finally {
            input.close();
        }
    } else {
        throw new IOException("Can not find meta file " + metaFilePath);
    }
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    MatrixMeta meta = matrixMetaManager.getMatrix(matrixLoadContext.getMatrixName());
    if (meta == null) {
        throw new IllegalStateException("Can not find matrix " + matrixLoadContext.getMatrixName());
    }
    Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
    Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
    for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
        ParameterServerId psId = partEntry.getValue().getMasterPs();
        if (psId == null) {
            throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
        }
        Set partIds = psIdToPartIdsMap.get(psId);
        if (partIds == null) {
            partIds = new HashSet();
            psIdToPartIdsMap.put(psId, partIds);
        }
        partIds.add(partEntry.getKey());
    }
    int matrixId = meta.getId();
    Map<ParameterServerId, PSMatrixLoadContext> ret = new HashMap<>(psIdToPartIdsMap.size());
    for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
        List<Integer> partIds = new ArrayList<>(entry.getValue());
        partIds.sort(new Comparator<Integer>() {

            @Override
            public int compare(Integer id1, Integer id2) {
                return id1 - id2;
            }
        });
        PSMatrixLoadContext psMatrixLoadContext = new PSMatrixLoadContext(matrixId, matrixPath.toString(), partIds, matrixFilesMeta.getFormatClassName());
        ret.put(entry.getKey(), psMatrixLoadContext);
    }
    return ret;
}

Example 14 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class ModelConverter method getMeta.

/**
 * Get model meta
 *
 * @param modelDir model save directory path
 * @return model meta
 * @throws IOException
 */
public static MatrixFilesMeta getMeta(String modelDir, Configuration conf) throws IOException {
    Path modelPath = new Path(modelDir);
    Path meteFilePath = new Path(modelPath, ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta meta = new MatrixFilesMeta();
    FileSystem fs = meteFilePath.getFileSystem(conf);
    if (!fs.exists(meteFilePath)) {
        throw new IOException("matrix meta file does not exist ");
    }
    FSDataInputStream input = fs.open(meteFilePath);
    meta.read(input);
    input.close();
    return meta;
}

Example 15 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class ModelMergeAndConvert method getMeta.

/**
 * Get model meta
 *
 * @param modelDir model save directory path
 * @return model meta
 * @throws IOException
 */
public static MatrixFilesMeta getMeta(String modelDir, Configuration conf) throws IOException {
    Path modelPath = new Path(modelDir);
    Path meteFilePath = new Path(modelPath, ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta meta = new MatrixFilesMeta();
    FileSystem fs = meteFilePath.getFileSystem(conf);
    if (!fs.exists(meteFilePath)) {
        throw new IOException("matrix meta file does not exist ");
    }
    FSDataInputStream input = fs.open(meteFilePath);
    meta.read(input);
    input.close();
    return meta;
}