Examples with ModelFilesMeta com.tencent.angel.model.output.format.ModelFilesMeta used on opensource projects

Example 1 with ModelFilesMeta

use of com.tencent.angel.model.output.format.ModelFilesMeta in project angel by Tencent.

the class AMMatrixCommitter method commitMatrix.

/**
 * Combine all output files of a model to a combine directory
 * @param matrixId matrix id
 * @param errorLogs error logs
 */
private void commitMatrix(int matrixId, Vector<String> errorLogs) {
    LOG.info("start commit matrix " + matrixId);
    // Init matrix files meta
    List<ParameterServerId> psIds = new ArrayList<>(context.getMatrixMetaManager().getMasterPsIds(matrixId));
    MatrixMeta meta = context.getMatrixMetaManager().getMatrix(matrixId);
    Map<String, String> kvMap = meta.getAttributes();
    ModelFilesMeta filesMeta = new ModelFilesMeta(matrixId, meta.getName(), meta.getRowType().getNumber(), meta.getRowNum(), meta.getColNum(), meta.getBlockRowNum(), meta.getBlockColNum(), kvMap);
    try {
        // Move output files
        Path srcPath = new Path(tmpOutputPath, ModelFilesConstent.resultDirName);
        Path destPath = new Path(tmpCombinePath, meta.getName());
        PartitionCommitOp partCommitOp = new PartitionCommitOp(srcPath, destPath, psIds, errorLogs, filesMeta, 0, psIds.size());
        fileOpExecutor.execute(partCommitOp);
        partCommitOp.join();
        // Write the meta file
        long startTs = System.currentTimeMillis();
        Path metaFile = new Path(destPath, ModelFilesConstent.modelMetaFileName);
        Path tmpMetaFile = HdfsUtil.toTmpPath(metaFile);
        FSDataOutputStream metaOut = fs.create(tmpMetaFile, (short) 1);
        filesMeta.write(metaOut);
        metaOut.flush();
        metaOut.close();
        HdfsUtil.rename(tmpMetaFile, metaFile, fs);
        LOG.info("commit meta file use time=" + (System.currentTimeMillis() - startTs));
    } catch (Throwable x) {
        errorLogs.add("move output files for matrix " + meta.getName() + " failed, error msg = " + x.getMessage());
        LOG.error("move output files for matrix " + meta.getName() + " failed.", x);
    }
}