Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 46 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MatrixOpLog method init.

public void init() {
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    matrix = createMatrix(meta);
}

Example 47 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MatrixOpLog method split.

/**
 * Split the update according to the matrix partitions
 *
 * @param psUpdateData partition -> row split list map
 */
public void split(Map<PartitionKey, List<RowUpdateSplit>> psUpdateData) {
    long startTime = System.currentTimeMillis();
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    int row = matrixMeta.getRowNum();
    boolean enableFilter = matrixMeta.getAttribute(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, MatrixConf.DEFAULT_MATRIX_OPLOG_ENABLEFILTER).equalsIgnoreCase("true");
    double filterThreshold = Double.valueOf(matrixMeta.getAttribute(MatrixConf.MATRIX_OPLOG_FILTER_THRESHOLD, MatrixConf.DEFAULT_MATRIX_OPLOG_FILTER_THRESHOLD));
    for (int rowId = 0; rowId < row; rowId++) {
        Vector vector = getRow(rowId);
        if (vector == null) {
            continue;
        }
        List<PartitionKey> partitions = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId, rowId);
        // Doing average or not
        if (matrixMeta.isAverage()) {
            vector.div(PSAgentContext.get().getTotalTaskNum());
        }
        // Filter un-important update
        if (enableFilter) {
            vector = vector.filter(filterThreshold);
        }
        // Split this row according the matrix partitions
        Map<PartitionKey, RowUpdateSplit> splits = RowUpdateSplitUtils.split(vector, partitions);
        // Set split context
        for (Map.Entry<PartitionKey, RowUpdateSplit> entry : splits.entrySet()) {
            RowUpdateSplitContext context = new RowUpdateSplitContext();
            context.setEnableFilter(enableFilter);
            context.setFilterThreshold(filterThreshold);
            context.setPartKey(entry.getKey());
            entry.getValue().setSplitContext(context);
            List<RowUpdateSplit> rowSplits = psUpdateData.get(entry.getKey());
            if (rowSplits == null) {
                rowSplits = new ArrayList<>();
                psUpdateData.put(entry.getKey(), rowSplits);
            }
            rowSplits.add(entry.getValue());
        }
        // Remove the row from matrix
        removeRow(rowId);
    }
    LOG.debug("taking " + (System.currentTimeMillis() - startTime) + " ms to split logs for matrix=" + matrixId);
}

Example 48 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class MatrixOpLog method flushToLocalStorage.

/**
 * Flush the update in cache to local matrix storage
 */
public void flushToLocalStorage() {
    MatrixStorage storage = PSAgentContext.get().getMatrixStorageManager().getMatrixStoage(matrixId);
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    int row = matrixMeta.getRowNum();
    Vector deltaVector;
    Vector vector;
    ReentrantReadWriteLock globalStorageLock = storage.getLock();
    try {
        globalStorageLock.writeLock().lock();
        for (int rowIndex = 0; rowIndex < row; rowIndex++) {
            deltaVector = getRow(rowIndex);
            vector = storage.getRow(rowIndex);
            if (deltaVector == null || vector == null) {
                continue;
            }
            vector.iaxpy(deltaVector, 1.0 / PSAgentContext.get().getTotalTaskNum());
        }
    } finally {
        globalStorageLock.writeLock().unlock();
    }
}

Example 49 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class GeneralGetUtils method partitionGet.

public static PartitionGetResult partitionGet(PSContext psContext, PartitionGetParam partParam) {
    GeneralPartGetParam param = (GeneralPartGetParam) partParam;
    KeyPart keyPart = param.getIndicesPart();
    // Long type node id
    long[] nodeIds = ((ILongKeyPartOp) keyPart).getKeys();
    ServerLongAnyRow row = GraphMatrixUtils.getPSLongKeyRow(psContext, param);
    // Get data
    IElement[] data = new IElement[nodeIds.length];
    for (int i = 0; i < nodeIds.length; i++) {
        data[i] = row.get(nodeIds[i]);
    }
    MatrixMeta meta = psContext.getMatrixMetaManager().getMatrixMeta(param.getMatrixId());
    try {
        return new PartGeneralGetResult(meta.getValueClass(), nodeIds, data);
    } catch (ClassNotFoundException e) {
        throw new AngelException("Can not get value class ");
    }
}

Example 50 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class AMModelSaver method getLoadPath.

private SaveResult getLoadPath(int matrixId) throws IOException {
    SaveResult result = matrixIdToLoadPathResult.get(matrixId);
    if (result == null) {
        String loadPath = context.getConf().get(AngelConf.ANGEL_LOAD_MODEL_PATH);
        if (loadPath != null) {
            AMMatrixMetaManager matrixMetaManager = context.getMatrixMetaManager();
            MatrixMeta meta = matrixMetaManager.getMatrix(matrixId);
            if (meta != null) {
                Path matrixPath = new Path(loadPath, meta.getName());
                FileSystem fs = matrixPath.getFileSystem(context.getConf());
                if (fs.exists(matrixPath)) {
                    result = new SaveResult(loadPath, matrixPath.toString(), -1);
                    matrixIdToLoadPathResult.putIfAbsent(matrixId, result);
                }
            }
        }
    }
    return result;
}