Examples with PSModelFilesMeta com.tencent.angel.model.output.format.PSModelFilesMeta used on opensource projects

Example 1 with PSModelFilesMeta

use of com.tencent.angel.model.output.format.PSModelFilesMeta in project angel by Tencent.

the class AMMatrixCommitter method commitPartitions.

/**
 *  Move all model output files generated by a PS to the combine directory
 * @param moveSrcPath source path
 * @param moveDestPath dest path
 * @param psId parameter server id
 * @param errorLogs error logs
 * @param matrixMeta model files meta
 */
private void commitPartitions(Path moveSrcPath, Path moveDestPath, ParameterServerId psId, Vector<String> errorLogs, ModelFilesMeta matrixMeta) {
    Path psPath = new Path(moveSrcPath, String.valueOf(psId));
    Path serverMatrixPath = new Path(psPath, matrixMeta.getMatrixName());
    Path psMetaFilePath = new Path(serverMatrixPath, ModelFilesConstent.psModelMetaFileName);
    try {
        FSDataInputStream input = fs.open(psMetaFilePath);
        PSModelFilesMeta serverMatrixMeta = new PSModelFilesMeta();
        serverMatrixMeta.read(input);
        input.close();
        fs.delete(psMetaFilePath, false);
        matrixMeta.merge(serverMatrixMeta);
        HdfsUtil.copyFilesInSameHdfs(serverMatrixPath, moveDestPath, fs);
        LOG.info("copy files of matrix " + matrixMeta.getMatrixName() + " from " + serverMatrixPath + " to " + moveDestPath + " success.");
    } catch (Throwable x) {
        errorLogs.add("copy files of matrix " + matrixMeta.getMatrixName() + " from " + serverMatrixPath + " to " + moveDestPath + " failed, error log is " + x.getMessage());
        LOG.error("copy files of matrix " + matrixMeta.getMatrixName() + " from " + serverMatrixPath + " to " + moveDestPath + " failed. ", x);
    }
}