Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 51 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class GetRowsTest method testSparseIntCompUDF.

public void testSparseIntCompUDF() throws Exception {
    Worker worker = LocalClusterContext.get().getWorker(workerAttempt0Id).getWorker();
    MatrixClient client1 = worker.getPSAgent().getMatrixClient(SPARSE_INT_MAT_COMP, 0);
    int matrixW1Id = client1.getMatrixId();
    MatrixMeta meta = worker.getPSAgent().getMatrix(SPARSE_INT_MAT_COMP);
    int[] index = genIndexs(feaNum, nnz);
    CompIntIntVector deltaVec = null;
    for (int rowId = 0; rowId < rowNum; rowId++) {
        long blockColNum = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta(matrixW1Id).getBlockColNum();
        int partNum = (feaNum + (int) blockColNum - 1) / (int) blockColNum;
        IntIntVector[] subVecs = new IntIntVector[partNum];
        for (int i = 0; i < partNum; i++) {
            subVecs[i] = VFactory.sparseIntVector((int) blockColNum);
        }
        deltaVec = new CompIntIntVector(feaNum, subVecs, (int) blockColNum);
        for (int i = 0; i < index.length; i++) {
            deltaVec.set(index[i], index[i]);
        }
        client1.increment(rowId, deltaVec, true);
    }
    int[] rowIds = new int[rowNum];
    for (int i = 0; i < rowNum; i++) {
        rowIds[i] = i;
    }
    Vector[] rows = client1.getRows(rowIds);
    for (int i = 0; i < rowNum; i++) {
        for (int id : index) {
            Assert.assertEquals(((CompIntIntVector) rows[i]).get(id), deltaVec.get(id), zero);
        }
        Assert.assertTrue(index.length == ((CompIntIntVector) rows[i]).size());
    }
}

Example 52 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class PSManagerTest method checkMatrixInfo.

private void checkMatrixInfo(ParameterServer ps, int w1Id, int w2Id, int w1Clock, int w2Clock) {
    PSMatrixMetaManager matrixPartManager = ps.getMatrixMetaManager();
    Map<Integer, MatrixMeta> matrixIdMap = matrixPartManager.getMatrixMetas();
    MatrixMeta sw1 = matrixIdMap.get(w1Id);
    MatrixMeta sw2 = matrixIdMap.get(w2Id);
    assertNotNull(sw1);
    assertNotNull(sw2);
    assertEquals(sw1.getPartitionMeta(0).getPartitionKey().getStartRow(), 0);
    assertEquals(sw1.getPartitionMeta(0).getPartitionKey().getEndRow(), 1);
    assertEquals(sw1.getPartitionMeta(0).getPartitionKey().getStartCol(), 0);
    assertEquals(sw1.getPartitionMeta(0).getPartitionKey().getEndCol(), 50000);
    assertEquals(sw1.getPartitionMeta(0).getPartitionKey().getMatrixId(), w1Id);
    assertEquals(sw1.getPartitionMeta(0).getPartitionKey().getPartitionId(), 0);
    assertEquals(sw1.getPartitionMeta(1).getPartitionKey().getStartRow(), 0);
    assertEquals(sw1.getPartitionMeta(1).getPartitionKey().getEndRow(), 1);
    assertEquals(sw1.getPartitionMeta(1).getPartitionKey().getStartCol(), 50000);
    assertEquals(sw1.getPartitionMeta(1).getPartitionKey().getEndCol(), 100000);
    assertEquals(sw1.getPartitionMeta(1).getPartitionKey().getMatrixId(), w1Id);
    assertEquals(sw1.getPartitionMeta(1).getPartitionKey().getPartitionId(), 1);
    assertEquals(sw2.getPartitionMeta(0).getPartitionKey().getStartRow(), 0);
    assertEquals(sw2.getPartitionMeta(0).getPartitionKey().getEndRow(), 1);
    assertEquals(sw2.getPartitionMeta(0).getPartitionKey().getStartCol(), 0);
    assertEquals(sw2.getPartitionMeta(0).getPartitionKey().getEndCol(), 50000);
    assertEquals(sw2.getPartitionMeta(0).getPartitionKey().getMatrixId(), w2Id);
    assertEquals(sw2.getPartitionMeta(0).getPartitionKey().getPartitionId(), 0);
    assertEquals(sw2.getPartitionMeta(1).getPartitionKey().getStartRow(), 0);
    assertEquals(sw2.getPartitionMeta(1).getPartitionKey().getEndRow(), 1);
    assertEquals(sw2.getPartitionMeta(1).getPartitionKey().getStartCol(), 50000);
    assertEquals(sw2.getPartitionMeta(1).getPartitionKey().getEndCol(), 100000);
    assertEquals(sw2.getPartitionMeta(1).getPartitionKey().getMatrixId(), w2Id);
    assertEquals(sw2.getPartitionMeta(1).getPartitionKey().getPartitionId(), 1);
}

Example 53 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class AMModelSaver method combineMatrix.

/**
 * Combine all output files of a model to a combine directory
 *
 * @param matrixContext matrix save context
 * @param errorLogs error logs
 */
private void combineMatrix(ModelSaveContext saveContext, MatrixSaveContext matrixContext, Vector<String> errorLogs, Path tmpCombinePath, FileSystem fs) {
    LOG.info("start commit matrix " + matrixContext.getMatrixName());
    // Init matrix files meta
    int matrixId = context.getMatrixMetaManager().getMatrix(matrixContext.getMatrixName()).getId();
    List<ParameterServerId> psIds = new ArrayList<>(context.getMatrixMetaManager().getMasterPsIds(matrixId));
    MatrixMeta meta = context.getMatrixMetaManager().getMatrix(matrixId);
    Map<String, String> kvMap = meta.getAttributes();
    MatrixFilesMeta filesMeta = new MatrixFilesMeta(matrixId, meta.getName(), matrixContext.getFormatClassName(), meta.getRowType().getNumber(), meta.getRowNum(), meta.getColNum(), meta.getBlockRowNum(), meta.getBlockColNum(), kvMap);
    filesMeta.setFeatureIndexStart(meta.getIndexStart());
    filesMeta.setFeatureIndexEnd(meta.getIndexEnd());
    try {
        // Move output files
        Path srcPath = new Path(saveContext.getTmpSavePath(), ModelFilesConstent.resultDirName);
        Path destPath = new Path(tmpCombinePath, meta.getName());
        PSModelCombineOp partCombineOp = new PSModelCombineOp(srcPath, destPath, psIds, errorLogs, filesMeta, 0, psIds.size(), fs);
        fileOpExecutor.execute(partCombineOp);
        partCombineOp.join();
        // Write the meta file
        long startTs = System.currentTimeMillis();
        Path metaFile = new Path(destPath, ModelFilesConstent.modelMetaFileName);
        Path tmpMetaFile = HdfsUtil.toTmpPath(metaFile);
        FSDataOutputStream metaOut = fs.create(tmpMetaFile);
        filesMeta.write(metaOut);
        metaOut.flush();
        metaOut.close();
        HdfsUtil.rename(tmpMetaFile, metaFile, fs);
        LOG.info("commit meta file use time=" + (System.currentTimeMillis() - startTs));
    } catch (Throwable x) {
        errorLogs.add("move output files for matrix " + meta.getName() + " failed, error msg = " + x.getMessage());
        LOG.error("move output files for matrix " + meta.getName() + " failed.", x);
    }
}

Example 54 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class AMModelLoader method split.

private Map<ParameterServerId, PSMatrixLoadContext> split(MatrixLoadContext matrixLoadContext, ModelLoadContext modelLoadContext) throws IOException {
    Path matrixPath = new Path(modelLoadContext.getLoadPath(), matrixLoadContext.getMatrixName());
    Path metaFilePath = new Path(matrixPath, ModelFilesConstent.modelMetaFileName);
    MatrixFilesMeta matrixFilesMeta = new MatrixFilesMeta();
    FileSystem fs = metaFilePath.getFileSystem(context.getConf());
    if (fs.exists(metaFilePath)) {
        FSDataInputStream input = fs.open(metaFilePath);
        try {
            matrixFilesMeta.read(input);
        } catch (Throwable e) {
            throw new IOException("Read matrix meta failed ", e);
        } finally {
            input.close();
        }
    } else {
        throw new IOException("Can not find meta file " + metaFilePath);
    }
    AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
    MatrixMeta meta = matrixMetaManager.getMatrix(matrixLoadContext.getMatrixName());
    if (meta == null) {
        throw new IllegalStateException("Can not find matrix " + matrixLoadContext.getMatrixName());
    }
    Map<Integer, PartitionMeta> partitions = meta.getPartitionMetas();
    Map<ParameterServerId, Set<Integer>> psIdToPartIdsMap = new HashMap<>();
    for (Map.Entry<Integer, PartitionMeta> partEntry : partitions.entrySet()) {
        ParameterServerId psId = partEntry.getValue().getMasterPs();
        if (psId == null) {
            throw new IllegalStateException("Can not get ps for partition " + partEntry.getKey());
        }
        Set partIds = psIdToPartIdsMap.get(psId);
        if (partIds == null) {
            partIds = new HashSet();
            psIdToPartIdsMap.put(psId, partIds);
        }
        partIds.add(partEntry.getKey());
    }
    int matrixId = meta.getId();
    Map<ParameterServerId, PSMatrixLoadContext> ret = new HashMap<>(psIdToPartIdsMap.size());
    for (Map.Entry<ParameterServerId, Set<Integer>> entry : psIdToPartIdsMap.entrySet()) {
        List<Integer> partIds = new ArrayList<>(entry.getValue());
        partIds.sort(new Comparator<Integer>() {

            @Override
            public int compare(Integer id1, Integer id2) {
                return id1 - id2;
            }
        });
        PSMatrixLoadContext psMatrixLoadContext = new PSMatrixLoadContext(matrixId, matrixPath.toString(), partIds, matrixFilesMeta.getFormatClassName());
        ret.put(entry.getKey(), psMatrixLoadContext);
    }
    return ret;
}

Example 55 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseFloatCompVector.

public static CompLongFloatVector mergeSparseFloatCompVector(LongIndexGetParam param, List<PartitionGetResult> partResults) {
    Map<PartitionKey, PartitionGetResult> partKeyToResultMap = mapPartKeyToResult(partResults);
    List<PartitionKey> partKeys = getSortedPartKeys(param.matrixId, param.getRowId());
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.matrixId);
    long dim = meta.getColNum();
    long subDim = meta.getBlockColNum();
    int size = partKeys.size();
    LongFloatVector[] splitVecs = new LongFloatVector[size];
    for (int i = 0; i < size; i++) {
        if (param.getPartKeyToIndexesMap().containsKey(partKeys.get(i))) {
            float[] values = ((IndexPartGetFloatResult) partKeyToResultMap.get(partKeys.get(i))).getValues();
            long[] indices = param.getPartKeyToIndexesMap().get(partKeys.get(i));
            transformIndices(indices, partKeys.get(i));
            splitVecs[i] = VFactory.sparseLongKeyFloatVector(subDim, indices, values);
        } else {
            splitVecs[i] = VFactory.sparseLongKeyFloatVector(subDim, 1);
        }
    }
    CompLongFloatVector vector = VFactory.compLongFloatVector(dim, splitVecs, subDim);
    vector.setMatrixId(param.getMatrixId());
    vector.setRowId(param.getRowId());
    return vector;
}