Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 16 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseLongCompVector.

public static CompLongLongVector mergeSparseLongCompVector(LongIndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    long dim = meta.getColNum();
    long subDim = meta.getBlockColNum();
    int size = partKeys.size();
    LongLongVector[] splitVecs = new LongLongVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            long[] values = ((IndexPartGetLongResult) partKeyToResultMap.get(partKeys.get(i))).getValues();
            long[] indices = param.getPartKeyToIndexesMap().get(partKeys.get(i));
            transformIndices(indices, partKeys.get(i));
            splitVecs[i] = VFactory.sparseLongKeyLongVector(subDim, indices, values);
        } else {
            splitVecs[i] = VFactory.sparseLongKeyLongVector(subDim, 0);
        }
    }
    CompLongLongVector vector = VFactory.compLongLongVector(dim, splitVecs, subDim);
    vector.setMatrixId(param.getMatrixId());
    vector.setRowId(param.getRowId());
    return vector;
}

Example 17 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseIntCompVector.

public static CompIntIntVector mergeSparseIntCompVector(IndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    int dim = (int) meta.getColNum();
    int subDim = (int) meta.getBlockColNum();
    int size = partKeys.size();
    IntIntVector[] splitVecs = new IntIntVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            int[] values = ((IndexPartGetIntResult) partKeyToResultMap.get(partKeys.get(i))).getValues();
            int[] indices = param.getPartKeyToIndexesMap().get(partKeys.get(i));
            transformIndices(indices, partKeys.get(i));
            splitVecs[i] = VFactory.sparseIntVector(subDim, indices, values);
        } else {
            splitVecs[i] = VFactory.sparseIntVector(subDim, 0);
        }
    }
    CompIntIntVector vector = VFactory.compIntIntVector(dim, splitVecs, subDim);
    vector.setMatrixId(param.getMatrixId());
    vector.setRowId(param.getRowId());
    return vector;
}

Example 18 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseDoubleCompVector.

public static CompLongDoubleVector mergeSparseDoubleCompVector(LongIndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    long dim = meta.getColNum();
    long subDim = meta.getBlockColNum();
    int size = partKeys.size();
    LongDoubleVector[] splitVecs = new LongDoubleVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            double[] values = ((IndexPartGetDoubleResult) partKeyToResultMap.get(partKeys.get(i))).getValues();
            long[] indices = param.getPartKeyToIndexesMap().get(partKeys.get(i));
            transformIndices(indices, partKeys.get(i));
            splitVecs[i] = VFactory.sparseLongKeyDoubleVector(subDim, indices, values);
        } else {
            splitVecs[i] = VFactory.sparseLongKeyDoubleVector(subDim, 1);
        }
    }
    CompLongDoubleVector vector = VFactory.compLongDoubleVector(dim, splitVecs, subDim);
    vector.setMatrixId(param.getMatrixId());
    vector.setRowId(param.getRowId());
    return vector;
}

Example 19 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ProtobufUtil method convertToMatrixMeta.

public static MatrixMeta convertToMatrixMeta(MatrixMetaProto matrixMetaProto) throws ClassNotFoundException {
    MatrixContext matrixContext = convertToMatrixContext(matrixMetaProto.getMatrixContext());
    List<PartitionMetaProto> partMetaProtos = matrixMetaProto.getPartMetasList();
    int size = partMetaProtos.size();
    Map<Integer, PartitionMeta> partitionMetas = new HashMap<>(size);
    for (int i = 0; i < size; i++) {
        partitionMetas.put(partMetaProtos.get(i).getPartitionId(), convertToParitionMeta(matrixContext.getMatrixId(), partMetaProtos.get(i)));
    }
    return new MatrixMeta(matrixMetaProto.getTotalPartNum(), matrixContext, partitionMetas);
}

Example 20 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class SnapshotDumper method checkpoint.

private void checkpoint(List<Integer> matrixIds, int checkpointId, boolean cloneFirst, boolean sortFirst) throws IOException {
    if (matrixIds == null || matrixIds.isEmpty()) {
        return;
    }
    List<PSMatrixSaveContext> saveContexts = new ArrayList<>(matrixIds.size());
    List<Path> checkpointItemPaths = new ArrayList<>(matrixIds.size());
    List<Path> tempPaths = new ArrayList<>(matrixIds.size());
    for (int matrixId : matrixIds) {
        Path checkpointItemPath = genCheckpointPath(matrixId, checkpointId);
        Path tempPath = genTmpCheckpointPath(checkpointItemPath);
        checkpointItemPaths.add(checkpointItemPath);
        tempPaths.add(tempPath);
        MatrixMeta meta = context.getMatrixMetaManager().getMatrixMeta(matrixId);
        saveContexts.add(new PSMatrixSaveContext(matrixId, new ArrayList<>(meta.getPartitionMetas().keySet()), null, SnapshotFormat.class.getName(), tempPath.toString(), cloneFirst, sortFirst));
    }
    context.getIOExecutors().save(new PSMatricesSaveContext(-1, -1, saveContexts), dumpParallel);
    // Rename temp to item path
    FileSystem fs = baseDirPath.getFileSystem(context.getConf());
    for (int i = 0; i < matrixIds.size(); i++) {
        HdfsUtil.rename(tempPaths.get(i), checkpointItemPaths.get(i), fs);
        clearOldCheckpoint(fs, genMatrixPath(matrixIds.get(0)));
    }
}