Examples with MatrixFilesMeta com.tencent.angel.model.output.format.MatrixFilesMeta used on opensource projects

Example 1 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class AMMatrixMetaManager method loadPartitionInfoFromHDFS.

/**
 * Load matrix proto from hdfs.
 *
 * @param path the path
 * @param conf the conf
 * @return matrix partitions
 * @throws IOException the io exception
 */
private List<PartitionMeta> loadPartitionInfoFromHDFS(String path, MatrixContext matrixContext, Configuration conf) throws IOException {
    Path meteFilePath = new Path(new Path(path, matrixContext.getName()), ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta meta = new MatrixFilesMeta();
    FileSystem fs = meteFilePath.getFileSystem(conf);
    LOG.info("Load matrix meta for matrix " + matrixContext.getName());
    if (!fs.exists(meteFilePath)) {
        throw new IOException("matrix meta file does not exist ");
    }
    FSDataInputStream input = fs.open(meteFilePath);
    try {
        meta.read(input);
    } catch (Throwable e) {
        throw new IOException("Read meta failed ", e);
    } finally {
        input.close();
    }
    List<PartitionMeta> matrixPartitions = new ArrayList<>();
    Map<Integer, MatrixPartitionMeta> partMetas = meta.getPartMetas();
    int matrixId = 0;
    try {
        writeLock.lock();
        matrixId = maxMatrixId++;
    } finally {
        writeLock.unlock();
    }
    for (Map.Entry<Integer, MatrixPartitionMeta> partMetaEntry : partMetas.entrySet()) {
        matrixPartitions.add(new PartitionMeta(matrixId, partMetaEntry.getKey(), partMetaEntry.getValue().getStartRow(), partMetaEntry.getValue().getEndRow(), partMetaEntry.getValue().getStartCol(), partMetaEntry.getValue().getEndCol()));
    }
    return matrixPartitions;
}

Example 2 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class MatrixContext method loadMatrixMetaFromFile.

private void loadMatrixMetaFromFile(String name, String path, Configuration conf) throws IOException {
    Path meteFilePath = new Path(new Path(path, name), ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta meta = new MatrixFilesMeta();
    FileSystem fs = meteFilePath.getFileSystem(conf);
    LOG.info("Load matrix meta for matrix " + name + " from " + meteFilePath);
    if (!fs.exists(meteFilePath)) {
        throw new IOException("matrix meta file does not exist ");
    }
    FSDataInputStream input = fs.open(meteFilePath);
    try {
        meta.read(input);
    } catch (Throwable e) {
        throw new IOException("Read meta failed ", e);
    } finally {
        input.close();
    }
    rowNum = meta.getRow();
    colNum = meta.getCol();
    maxRowNumInBlock = meta.getBlockRow();
    maxColNumInBlock = meta.getBlockCol();
    indexStart = meta.getFeatureIndexStart();
    indexEnd = meta.getFeatureIndexEnd();
    rowType = RowType.valueOf(meta.getRowType());
    Map<String, String> oldAttributes = meta.getOptions();
    if (oldAttributes != null && !oldAttributes.isEmpty()) {
        for (Map.Entry<String, String> kv : oldAttributes.entrySet()) {
            if (kv.getKey().equals(MatrixConf.MATRIX_LOAD_PATH)) {
                continue;
            }
            attributes.put(kv.getKey(), kv.getValue());
        }
    }
}

Example 3 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class ModelMergeAndConvert method convert.

/**
 * Convert a angel model to other format
 *
 * @param conf           application configuration
 * @param modelInputDir  the directory of angel model files
 * @param modelOutputDir the save directory of converted model files
 * @param lineConvert    format serializer
 * @throws IOException
 */
public static void convert(Configuration conf, String modelInputDir, String modelOutputDir, ModelLineConvert lineConvert) throws IOException {
    // Load model meta
    MatrixFilesMeta meta = getMeta(modelInputDir, conf);
    // Convert model
    convertModel(conf, modelInputDir, modelOutputDir, meta, lineConvert);
}

Example 4 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class ModelConverter method convert.

/**
 * Convert a angel model to other format
 *
 * @param conf           application configuration
 * @param modelInputDir  the directory of angel model files
 * @param modelOutputDir the save directory of converted model files
 * @param lineConvert    format serializer
 * @throws IOException
 */
public static void convert(Configuration conf, String modelInputDir, String modelOutputDir, ModelLineConvert lineConvert) throws IOException {
    // Load model meta
    MatrixFilesMeta meta = getMeta(modelInputDir, conf);
    // Convert model
    convertModel(conf, modelInputDir, modelOutputDir, meta, lineConvert);
}

Example 5 with MatrixFilesMeta

use of com.tencent.angel.model.output.format.MatrixFilesMeta in project angel by Tencent.

the class ModelLoader method loadToIntMaps.

/**
 * Load dense double model to int->int maps
 *
 * @param modelDir model save directory path
 * @return model data
 */
public static Int2IntOpenHashMap[] loadToIntMaps(String modelDir, Configuration conf) throws IOException {
    // Load model meta
    MatrixFilesMeta meta = getMeta(modelDir, conf);
    RowType rowType = RowType.valueOf(meta.getRowType());
    // Check row type
    if (rowType != RowType.T_INT_SPARSE && rowType != RowType.T_INT_SPARSE_COMPONENT) {
        throw new IOException("model row type is not sparse int, you should check it");
    }
    // Load model
    SparseIntModel model = new SparseIntModel(meta.getRow(), meta.getCol());
    loadModel(modelDir, model, meta, conf);
    return model.getModel();
}