Examples with RowType com.tencent.angel.ml.matrix.RowType used on opensource projects

Example 31 with RowType

use of com.tencent.angel.ml.matrix.RowType in project angel by Tencent.

the class ModelMergeAndConvert method convertHeader.

private static void convertHeader(MatrixFilesMeta meta, FSDataOutputStream output) throws IOException {
    LOG.info("model meta=" + meta);
    output.writeBytes("modelName=" + meta.getMatrixName() + "\n");
    output.writeBytes("row=" + meta.getRow() + "\n");
    output.writeBytes("column=" + meta.getCol() + "\n");
    RowType rowType = RowType.valueOf(meta.getRowType());
    output.writeBytes("rowType=" + rowType.toString() + "\n");
}

Example 32 with RowType

use of com.tencent.angel.ml.matrix.RowType in project angel by Tencent.

the class ModelLoader method loadToFloatArrays.

/**
 * Load dense float model to a 2-dimension float array
 *
 * @param modelDir model save directory path
 * @return model data
 */
public static float[][] loadToFloatArrays(String modelDir, Configuration conf) throws IOException {
    // Load model meta
    MatrixFilesMeta meta = getMeta(modelDir, conf);
    RowType rowType = RowType.valueOf(meta.getRowType());
    // Check row type
    if (rowType != RowType.T_FLOAT_DENSE && rowType != RowType.T_FLOAT_DENSE_COMPONENT) {
        throw new IOException("model row type is not dense float, you should check it");
    }
    // Load model
    DenseFloatModel model = new DenseFloatModel(meta.getRow(), meta.getCol());
    loadModel(modelDir, model, meta, conf);
    return model.getModel();
}

Example 33 with RowType

use of com.tencent.angel.ml.matrix.RowType in project angel by Tencent.

the class RowSplitCombineUtils method combineRowSplitsPipeline.

/**
 * Combine the row splits use pipeline mode.
 *
 * @param cache the result cache for GET_ROW sub-requests
 * @param matrixId matrix id
 * @param rowIndex row index
 * @return TVector merged row
 * @throws InterruptedException interrupted while waiting for row splits
 */
public static TVector combineRowSplitsPipeline(GetRowPipelineCache cache, int matrixId, int rowIndex) throws InterruptedException {
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    RowType rowType = matrixMeta.getRowType();
    switch(rowType) {
        case T_DOUBLE_DENSE:
            return combineDenseDoubleRowSplits(cache, matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE:
            return combineServerSparseDoubleRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE_LONGKEY:
            return combineServerSparseDoubleLongKeyRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_FLOAT_SPARSE:
            return combineServerSparseFloatRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_INT_DENSE:
            return combineDenseIntRowSplits(cache, matrixMeta, rowIndex);
        case T_FLOAT_DENSE:
            return combineDenseFloatRowSplits(cache, matrixMeta, rowIndex);
        case T_INT_SPARSE:
            return combineServerSparseIntRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE_COMPONENT:
            return combineComponentServerSparseDoubleRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_FLOAT_SPARSE_COMPONENT:
            return combineComponentServerSparseFloatRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_INT_SPARSE_COMPONENT:
            return combineComponentServerSparseIntRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        case T_DOUBLE_SPARSE_LONGKEY_COMPONENT:
            return combineCompServerSparseDoubleLongKeyRowSplits(getAllRowSplitsFromCache(cache), matrixMeta, rowIndex);
        default:
            return null;
    }
}

Example 34 with RowType

use of com.tencent.angel.ml.matrix.RowType in project angel by Tencent.

the class IndexGetFunc method merge.

@Override
public GetResult merge(List<PartitionGetResult> partResults) {
    long startTs = System.currentTimeMillis();
    RowType rowType = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.getMatrixId()).getRowType();
    GetRowResult result = null;
    switch(rowType) {
        case T_DOUBLE_DENSE:
        case T_DOUBLE_SPARSE:
            result = new GetRowResult(ResponseType.SUCCESS, ValuesCombineUtils.mergeSparseDoubleVector((IndexGetParam) param, partResults));
            break;
        case T_DOUBLE_SPARSE_COMPONENT:
            result = new GetRowResult(ResponseType.SUCCESS, ValuesCombineUtils.mergeSparseDoubleCompVector((IndexGetParam) param, partResults));
            break;
        case T_FLOAT_DENSE:
        case T_FLOAT_SPARSE:
            result = new GetRowResult(ResponseType.SUCCESS, ValuesCombineUtils.mergeSparseFloatVector((IndexGetParam) param, partResults));
            break;
        case T_FLOAT_SPARSE_COMPONENT:
            result = new GetRowResult(ResponseType.SUCCESS, ValuesCombineUtils.mergeSparseFloatCompVector((IndexGetParam) param, partResults));
            break;
        case T_INT_DENSE:
        case T_INT_SPARSE:
            result = new GetRowResult(ResponseType.SUCCESS, ValuesCombineUtils.mergeSparseIntVector((IndexGetParam) param, partResults));
            break;
        case T_INT_SPARSE_COMPONENT:
            result = new GetRowResult(ResponseType.SUCCESS, ValuesCombineUtils.mergeSparseIntCompVector((IndexGetParam) param, partResults));
            break;
        default:
            throw new UnsupportedOperationException("Unsupport operation: update " + rowType + " to " + this.getClass().getName());
    }
    LOG.info("Merge use time=" + (System.currentTimeMillis() - startTs) + " ms");
    return result;
}

Example 35 with RowType

use of com.tencent.angel.ml.matrix.RowType in project angel by Tencent.

the class IndexGetFunc method partitionGet.

/**
 * Each server partition execute this function and return values of specified index.
 * @param partParam the partition parameter
 * @return values of specified index
 */
@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    long startTs = System.currentTimeMillis();
    ServerPartition part = psContext.getMatrixStorageManager().getPart(partParam.getMatrixId(), partParam.getPartKey().getPartitionId());
    PartitionGetResult result = null;
    if (part != null) {
        int rowId = ((IndexPartGetParam) partParam).getRowId();
        int[] indexes = ((IndexPartGetParam) partParam).getIndexes();
        ServerRow row = part.getRow(rowId);
        RowType rowType = row.getRowType();
        switch(rowType) {
            case T_DOUBLE_DENSE:
            case T_DOUBLE_SPARSE:
            case T_DOUBLE_SPARSE_COMPONENT:
                {
                    result = new IndexPartGetDoubleResult(partParam.getPartKey(), ((ServerDoubleRow) row).getValues(indexes));
                    break;
                }
            case T_FLOAT_DENSE:
            case T_FLOAT_SPARSE:
            case T_FLOAT_SPARSE_COMPONENT:
                {
                    result = new IndexPartGetFloatResult(partParam.getPartKey(), ((ServerFloatRow) row).getValues(indexes));
                    break;
                }
            case T_INT_DENSE:
            case T_INT_SPARSE:
            case T_INT_SPARSE_COMPONENT:
                {
                    result = new IndexPartGetIntResult(partParam.getPartKey(), ((ServerIntRow) row).getValues(indexes));
                    break;
                }
            default:
                throw new UnsupportedOperationException("Unsupport operation: update " + rowType + " to " + this.getClass().getName());
        }
    }
    LOG.info("Partition get use time=" + (System.currentTimeMillis() - startTs) + " ms");
    return result;
}