Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 6 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class AMMatrixCommitter method commitMatrix.

/**
 * Combine all output files of a model to a combine directory
 * @param matrixId matrix id
 * @param errorLogs error logs
 */
private void commitMatrix(int matrixId, Vector<String> errorLogs) {
    LOG.info("start commit matrix " + matrixId);
    // Init matrix files meta
    List<ParameterServerId> psIds = new ArrayList<>(context.getMatrixMetaManager().getMasterPsIds(matrixId));
    MatrixMeta meta = context.getMatrixMetaManager().getMatrix(matrixId);
    Map<String, String> kvMap = meta.getAttributes();
    ModelFilesMeta filesMeta = new ModelFilesMeta(matrixId, meta.getName(), meta.getRowType().getNumber(), meta.getRowNum(), meta.getColNum(), meta.getBlockRowNum(), meta.getBlockColNum(), kvMap);
    try {
        // Move output files
        Path srcPath = new Path(tmpOutputPath, ModelFilesConstent.resultDirName);
        Path destPath = new Path(tmpCombinePath, meta.getName());
        PartitionCommitOp partCommitOp = new PartitionCommitOp(srcPath, destPath, psIds, errorLogs, filesMeta, 0, psIds.size());
        fileOpExecutor.execute(partCommitOp);
        partCommitOp.join();
        // Write the meta file
        long startTs = System.currentTimeMillis();
        Path metaFile = new Path(destPath, ModelFilesConstent.modelMetaFileName);
        Path tmpMetaFile = HdfsUtil.toTmpPath(metaFile);
        FSDataOutputStream metaOut = fs.create(tmpMetaFile, (short) 1);
        filesMeta.write(metaOut);
        metaOut.flush();
        metaOut.close();
        HdfsUtil.rename(tmpMetaFile, metaFile, fs);
        LOG.info("commit meta file use time=" + (System.currentTimeMillis() - startTs));
    } catch (Throwable x) {
        errorLogs.add("move output files for matrix " + meta.getName() + " failed, error msg = " + x.getMessage());
        LOG.error("move output files for matrix " + meta.getName() + " failed.", x);
    }
}

Example 7 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseFloatCompVector.

public static CompSparseFloatVector mergeSparseFloatCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int size = partKeys.size();
    SparseFloatVector[] splitVecs = new SparseFloatVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            splitVecs[i] = new SparseFloatVector((int) meta.getColNum(), param.getPartKeyToIndexesMap().get(partKeys.get(i)), ((IndexPartGetFloatResult) partKeyToResultMap.get(partKeys.get(i))).getValues());
        }
    }
    return new CompSparseFloatVector(meta.getId(), param.getRowId(), (int) meta.getColNum(), partKeys.toArray(new PartitionKey[0]), splitVecs);
}

Example 8 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseDoubleCompVector.

public static CompSparseLongKeyDoubleVector mergeSparseDoubleCompVector(LongIndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int size = partKeys.size();
    SparseLongKeyDoubleVector[] splitVecs = new SparseLongKeyDoubleVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            splitVecs[i] = new SparseLongKeyDoubleVector(meta.getColNum(), param.getPartKeyToIndexesMap().get(partKeys.get(i)), ((LongIndexGetResult) partKeyToResultMap.get(partKeys.get(i))).getValues());
        }
    }
    CompSparseLongKeyDoubleVector vector = new CompSparseLongKeyDoubleVector(meta.getId(), param.getRowId(), meta.getColNum(), partKeys.toArray(new PartitionKey[0]), splitVecs);
    return vector;
}

Example 9 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ServerMatrix method init.

public void init() {
    MatrixMeta matrixMeta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
    Map<Integer, PartitionMeta> partMetas = matrixMeta.getPartitionMetas();
    String sourceClass = matrixMeta.getAttribute(AngelConf.ANGEL_PS_PARTITION_SOURCE_CLASS, AngelConf.DEFAULT_ANGEL_PS_PARTITION_SOURCE_CLASS);
    // Get server partition class
    Class<? extends IServerPartition> partClass;
    try {
        partClass = matrixMeta.getPartitionClass();
        // If partition class is not set, just use the default partition class
        if (partClass == null) {
            partClass = MatrixConf.DEFAULT_SERVER_PARTITION_CLASS;
        }
    } catch (Throwable e) {
        LOG.fatal("Server partition class failed ", e);
        throw new RuntimeException(e);
    }
    // Get server partition storage class type
    Class<? extends IServerPartitionStorage> storageClass;
    try {
        storageClass = matrixMeta.getPartitionStorageClass();
    } catch (Throwable e) {
        LOG.fatal("Server partition class failed ", e);
        throw new RuntimeException(e);
    }
    RowType rowType = matrixMeta.getRowType();
    // Get value class
    Class<? extends IElement> valueClass = null;
    if (rowType.isComplexValue()) {
        try {
            valueClass = matrixMeta.getValueClass();
        } catch (Throwable e) {
            LOG.fatal("Init value class failed ", e);
            throw new RuntimeException(e);
        }
        if (valueClass == null) {
            throw new RuntimeException("Complex type must set value type class!!");
        }
    }
    for (PartitionMeta partMeta : partMetas.values()) {
        ServerPartition part = ServerPartitionFactory.getPartition(partMeta.getPartitionKey(), partClass, storageClass, matrixMeta.getRowType(), valueClass, matrixMeta.getValidIndexNumInOnePart(), matrixMeta.isHash() ? RouterType.HASH : RouterType.RANGE);
        partitionMaps.put(partMeta.getPartId(), part);
        part.init();
        part.setState(PartitionState.READ_AND_WRITE);
    }
}

Example 10 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MasterService method getPSMatricesMeta.

@Override
public GetPSMatricesResponse getPSMatricesMeta(RpcController controller, GetPSMatricesMetaRequest request) throws ServiceException {
    Map<Integer, MatrixMeta> matrixIdToMetaMap = context.getMatrixMetaManager().getMatrixPartitions(ProtobufUtil.convertToId(request.getPsId()));
    GetPSMatricesResponse.Builder builder = GetPSMatricesResponse.newBuilder();
    if (matrixIdToMetaMap != null && !matrixIdToMetaMap.isEmpty()) {
        for (MatrixMeta meta : matrixIdToMetaMap.values()) {
            builder.addMatricesMeta(ProtobufUtil.convertToMatrixMetaProto(meta));
        }
    }
    return builder.build();
}