Examples with MatrixStorageManager com.tencent.angel.ps.storage.MatrixStorageManager used on opensource projects

Example 1 with MatrixStorageManager

use of com.tencent.angel.ps.storage.MatrixStorageManager in project angel by Tencent.

the class GetColumnFunc method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    if (partParam instanceof PartitionGetRowsParam) {
        PartitionGetRowsParam param = (PartitionGetRowsParam) partParam;
        PartitionKey pkey = param.getPartKey();
        pkey = psContext.getMatrixMetaManager().getMatrixMeta(pkey.getMatrixId()).getPartitionMeta(pkey.getPartitionId()).getPartitionKey();
        List<Integer> reqCols = param.getRowIndexes();
        int start = reqCols.get(0);
        int end = reqCols.get(1);
        MatrixStorageManager manager = psContext.getMatrixStorageManager();
        Map<Integer, Int2IntOpenHashMap> cks = new HashMap();
        for (int col = start; col < end; col++) cks.put(col, new Int2IntOpenHashMap());
        int rowOffset = pkey.getStartRow();
        int rowLength = pkey.getEndRow();
        for (int r = rowOffset; r < rowLength; r++) {
            ServerRow row = manager.getRow(pkey, r);
            if (row instanceof ServerIntIntRow) {
                for (int col = start; col < end; col++) {
                    Int2IntOpenHashMap map = cks.get(col);
                    int k = ((ServerIntIntRow) row).get(col);
                    if (k > 0)
                        map.put(row.getRowId(), k);
                }
            }
        }
        return new PartColumnResult(cks);
    } else {
        return null;
    }
}

Example 2 with MatrixStorageManager

use of com.tencent.angel.ps.storage.MatrixStorageManager in project angel by Tencent.

the class MatrixMetaManagerTest method testCreateMatrix.

@Test
public void testCreateMatrix() throws Exception {
    try {
        LOG.info("===========================testCreateMatrix===============================");
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        MasterClient masterClient = worker.getPSAgent().getMasterClient();
        int w3Id = -1;
        int w4Id = -1;
        // add matrix
        MatrixContext mMatrix = new MatrixContext();
        mMatrix.setName("w3");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, RowType.T_DOUBLE_DENSE.name());
        masterClient.createMatrix(mMatrix, 10000);
        mMatrix.setName("w4");
        mMatrix.setRowNum(1);
        mMatrix.setColNum(100000);
        mMatrix.setMaxRowNumInBlock(1);
        mMatrix.setMaxColNumInBlock(50000);
        mMatrix.setRowType(RowType.T_DOUBLE_DENSE);
        mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
        mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
        mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
        mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, RowType.T_DOUBLE_DENSE.name());
        masterClient.createMatrix(mMatrix, 10000);
        MatrixMeta w3Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w3");
        MatrixMeta w4Meta = worker.getPSAgent().getMatrixMetaManager().getMatrixMeta("w4");
        assertEquals(w3Meta.getRowNum(), 1);
        assertEquals(w3Meta.getColNum(), 100000);
        assertEquals(w3Meta.getRowType(), RowType.T_DOUBLE_DENSE);
        assertEquals(w4Meta.getRowNum(), 1);
        assertEquals(w4Meta.getColNum(), 100000);
        assertEquals(w4Meta.getRowType(), RowType.T_DOUBLE_DENSE);
        w3Id = w3Meta.getId();
        w4Id = w4Meta.getId();
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMMatrixMetaManager matrixMetaManager = angelAppMaster.getAppContext().getMatrixMetaManager();
        MatrixMeta matrixw3Proto = matrixMetaManager.getMatrix("w3");
        MatrixMeta matrixw4Proto = matrixMetaManager.getMatrix("w4");
        assertNotNull(matrixw3Proto);
        assertNotNull(matrixw4Proto);
        assertEquals(matrixw3Proto.getRowNum(), 1);
        assertEquals(matrixw3Proto.getColNum(), 100000);
        assertEquals(matrixw3Proto.getPartitionMetas().size(), 2);
        Map<Integer, PartitionMeta> w3Parts = matrixw3Proto.getPartitionMetas();
        assertEquals(w3Parts.get(0).getPss().get(0), psId);
        assertEquals(w3Parts.get(0).getPartId(), 0);
        assertEquals(w3Parts.get(0).getStartRow(), 0);
        assertEquals(w3Parts.get(0).getEndRow(), 1);
        assertEquals(w3Parts.get(0).getStartCol(), 0);
        assertEquals(w3Parts.get(0).getEndCol(), 50000);
        assertEquals(w3Parts.get(1).getPartId(), 1);
        assertEquals(w3Parts.get(1).getStartRow(), 0);
        assertEquals(w3Parts.get(1).getEndRow(), 1);
        assertEquals(w3Parts.get(1).getStartCol(), 50000);
        assertEquals(w3Parts.get(1).getEndCol(), 100000);
        Map<Integer, PartitionMeta> w4Parts = matrixw4Proto.getPartitionMetas();
        assertEquals(w4Parts.get(0).getPss().get(0), psId);
        assertEquals(w4Parts.get(0).getPartId(), 0);
        assertEquals(w4Parts.get(0).getStartRow(), 0);
        assertEquals(w4Parts.get(0).getEndRow(), 1);
        assertEquals(w4Parts.get(0).getStartCol(), 0);
        assertEquals(w4Parts.get(0).getEndCol(), 50000);
        assertEquals(w4Parts.get(1).getPartId(), 1);
        assertEquals(w4Parts.get(1).getStartRow(), 0);
        assertEquals(w4Parts.get(1).getEndRow(), 1);
        assertEquals(w4Parts.get(1).getStartCol(), 50000);
        assertEquals(w4Parts.get(1).getEndCol(), 100000);
        ParameterServer ps = LocalClusterContext.get().getPS(psAttempt0Id).getPS();
        PSMatrixMetaManager matrixPartManager = ps.getMatrixMetaManager();
        PartitionMeta w3Part0 = matrixPartManager.getPartMeta(w3Id, 0);
        PartitionMeta w3Part1 = matrixPartManager.getPartMeta(w3Id, 1);
        assertTrue(w3Part0 != null);
        assertTrue(w3Part1 != null);
        assertEquals(w3Part0.getPartitionKey().getStartRow(), 0);
        assertEquals(w3Part0.getPartitionKey().getEndRow(), 1);
        assertEquals(w3Part0.getPartitionKey().getStartCol(), 0);
        assertEquals(w3Part0.getPartitionKey().getEndCol(), 50000);
        assertEquals(w3Part1.getPartitionKey().getStartRow(), 0);
        assertEquals(w3Part1.getPartitionKey().getEndRow(), 1);
        assertEquals(w3Part1.getPartitionKey().getStartCol(), 50000);
        assertEquals(w3Part1.getPartitionKey().getEndCol(), 100000);
        PartitionMeta w4Part0 = matrixPartManager.getPartMeta(w4Id, 0);
        PartitionMeta w4Part1 = matrixPartManager.getPartMeta(w4Id, 1);
        assertTrue(w4Part0 != null);
        assertTrue(w4Part1 != null);
        assertEquals(w4Part0.getPartitionKey().getStartRow(), 0);
        assertEquals(w4Part0.getPartitionKey().getEndRow(), 1);
        assertEquals(w4Part0.getPartitionKey().getStartCol(), 0);
        assertEquals(w4Part0.getPartitionKey().getEndCol(), 50000);
        assertEquals(w4Part1.getPartitionKey().getStartRow(), 0);
        assertEquals(w4Part1.getPartitionKey().getEndRow(), 1);
        assertEquals(w4Part1.getPartitionKey().getStartCol(), 50000);
        assertEquals(w4Part1.getPartitionKey().getEndCol(), 100000);
        MatrixClient w4ClientForTask0 = worker.getPSAgent().getMatrixClient("w4", 0);
        MatrixClient w4ClientForTask1 = worker.getPSAgent().getMatrixClient("w4", 1);
        TaskContext task0Context = w4ClientForTask0.getTaskContext();
        TaskContext task1Context = w4ClientForTask1.getTaskContext();
        double[] delta = new double[100000];
        for (int i = 0; i < delta.length; i++) {
            delta[i] = 1.0;
        }
        int iterIndex = 0;
        while (iterIndex < 5) {
            IntDoubleVector row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
            double sum1 = sum(row1.getStorage().getValues());
            LOG.info("taskid=" + task0Context.getIndex() + ", matrixId=" + w4ClientForTask0.getMatrixId() + ", rowIndex=0, local row sum=" + sum1);
            IntDoubleVector deltaRow1 = new IntDoubleVector(delta.length, new IntDoubleDenseVectorStorage(delta));
            deltaRow1.setMatrixId(w4ClientForTask0.getMatrixId());
            deltaRow1.setRowId(0);
            w4ClientForTask0.increment(deltaRow1);
            w4ClientForTask0.clock().get();
            task0Context.increaseEpoch();
            IntDoubleVector row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
            double sum2 = sum(row2.getStorage().getValues());
            LOG.info("taskid=" + task1Context.getIndex() + ", matrixId=" + w4ClientForTask1.getMatrixId() + ", rowIndex=1, local row sum=" + sum2);
            IntDoubleVector deltaRow2 = new IntDoubleVector(delta.length, new IntDoubleDenseVectorStorage(delta));
            deltaRow2.setMatrixId(w4ClientForTask1.getMatrixId());
            deltaRow2.setRowId(0);
            w4ClientForTask1.increment(deltaRow2);
            w4ClientForTask1.clock().get();
            task1Context.increaseEpoch();
            iterIndex++;
        }
        AMTaskManager amTaskManager = angelAppMaster.getAppContext().getTaskManager();
        AMTask amTask0 = amTaskManager.getTask(task0Id);
        AMTask amTask1 = amTaskManager.getTask(task1Id);
        assertEquals(amTask0.getIteration(), 5);
        assertEquals(amTask1.getIteration(), 5);
        Int2IntOpenHashMap task0MatrixClocks = amTask0.getMatrixClocks();
        assertEquals(task0MatrixClocks.size(), 1);
        assertEquals(task0MatrixClocks.get(w4Id), 5);
        Int2IntOpenHashMap task1MatrixClocks = amTask1.getMatrixClocks();
        assertEquals(task1MatrixClocks.size(), 1);
        assertEquals(task1MatrixClocks.get(w4Id), 5);
        IntDoubleVector row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
        double sum1 = sum(row1.getStorage().getValues());
        assertEquals(sum1, 1000000.0, 0.000001);
        IntDoubleVector row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
        double sum2 = sum(row2.getStorage().getValues());
        assertEquals(sum2, 1000000.0, 0.000001);
        masterClient.releaseMatrix(w3Meta.getName());
        Thread.sleep(10000);
        matrixw3Proto = matrixMetaManager.getMatrix("w3");
        assertTrue(matrixw3Proto == null);
        MatrixStorageManager matrixStorageManager = LocalClusterContext.get().getPS(psAttempt0Id).getPS().getMatrixStorageManager();
        ServerMatrix sw3 = matrixStorageManager.getMatrix(w3Id);
        assertTrue(sw3 == null);
        w4ClientForTask0.clock().get();
        w4ClientForTask1.clock().get();
        row1 = (IntDoubleVector) w4ClientForTask0.getRow(0);
        sum1 = sum(row1.getStorage().getValues());
        assertEquals(sum1, 1000000.0, 0.000001);
        row2 = (IntDoubleVector) w4ClientForTask1.getRow(0);
        sum2 = sum(row2.getStorage().getValues());
        assertEquals(sum2, 1000000.0, 0.000001);
    } catch (Exception x) {
        LOG.error("run testCreateMatrix failed ", x);
        throw x;
    }
}

Example 3 with MatrixStorageManager

use of com.tencent.angel.ps.storage.MatrixStorageManager in project angel by Tencent.

the class ParameterServer method initialize.

/**
 * Initialize.
 */
public void initialize() throws IOException, InstantiationException, IllegalAccessException {
    LOG.info("Initialize a parameter server");
    ServerRow.maxLockWaitTimeMs = conf.getInt(AngelConf.ANGEL_PS_MAX_LOCK_WAITTIME_MS, AngelConf.DEFAULT_ANGEL_PS_MAX_LOCK_WAITTIME_MS);
    ServerRow.useAdaptiveKey = conf.getBoolean(AngelConf.ANGEL_PS_USE_ADAPTIVE_KEY_ENABLE, AngelConf.DEFAULT_ANGEL_PS_USE_ADAPTIVE_KEY_ENABLE);
    ServerRow.useAdaptiveStorage = conf.getBoolean(AngelConf.ANGEL_PS_USE_ADAPTIVE_STORAGE_ENABLE, AngelConf.DEFAULT_ANGEL_PS_USE_ADAPTIVE_STORAGE_ENABLE);
    ServerRow.sparseToDenseFactor = conf.getFloat(AngelConf.ANGEL_PS_SPARSE_TO_DENSE_FACTOR, AngelConf.DEFAULT_ANGEL_PS_SPARSE_TO_DENSE_FACTOR);
    dataCollectionInterval = conf.getInt(AngelConf.ANGEL_PS_HEARTBEAT_DATA_COLLECTION_INTERVAL, AngelConf.DEFAULT_ANGEL_PS_HEARTBEAT_DATA_COLLECTION_INTERVAL);
    locationManager = new PSLocationManager(context);
    locationManager.setMasterLocation(masterLocation);
    runningContext = new RunningContext(context);
    workerPool = new WorkerPool(context, runningContext);
    workerPool.init();
    ioExecutor = new PSModelIOExecutor(context);
    ioExecutor.init();
    matrixStorageManager = new MatrixStorageManager(context);
    matrixMetaManager = new PSMatrixMetaManager(context);
    master = new MasterClient(context);
    master.init();
    psServerService = new ParameterServerService(context);
    psServerService.start();
    matrixTransportServer = new MatrixTransportServer(getPort() + 1, context);
    saver = new PSModelSaver(context);
    loader = new PSModelLoader(context);
    snapshotDumper = new SnapshotDumper(context);
}

Example 4 with MatrixStorageManager

use of com.tencent.angel.ps.storage.MatrixStorageManager in project angel by Tencent.

the class MasterRecoverTest method checkMatrixInfo.

private void checkMatrixInfo(ParameterServer ps, int w1Id, int w2Id, int w1Clock, int w2Clock) {
    MatrixStorageManager matrixPartManager = ps.getMatrixStorageManager();
    ConcurrentHashMap<Integer, ServerMatrix> matrixIdMap = matrixPartManager.getMatrices();
    ServerMatrix sw1 = matrixIdMap.get(w1Id);
    ServerMatrix sw2 = matrixIdMap.get(w2Id);
    assertTrue(sw1 != null);
    assertTrue(sw2 != null);
    LOG.info("======================partition key is " + sw1.getPartition(0).getPartitionKey());
    LOG.info("======================partition key is " + sw1.getPartition(1).getPartitionKey());
    assertEquals(sw1.getPartition(0).getPartitionKey().getStartRow(), 0);
    assertEquals(sw1.getPartition(0).getPartitionKey().getEndRow(), 1);
    assertEquals(sw1.getPartition(0).getPartitionKey().getStartCol(), 0);
    assertEquals(sw1.getPartition(0).getPartitionKey().getEndCol(), 50000);
    assertEquals(sw1.getPartition(0).getPartitionKey().getMatrixId(), w1Id);
    assertEquals(sw1.getPartition(0).getPartitionKey().getPartitionId(), 0);
    assertEquals(sw1.getPartition(1).getPartitionKey().getStartRow(), 0);
    assertEquals(sw1.getPartition(1).getPartitionKey().getEndRow(), 1);
    assertEquals(sw1.getPartition(1).getPartitionKey().getStartCol(), 50000);
    assertEquals(sw1.getPartition(1).getPartitionKey().getEndCol(), 100000);
    assertEquals(sw1.getPartition(1).getPartitionKey().getMatrixId(), w1Id);
    assertEquals(sw1.getPartition(1).getPartitionKey().getPartitionId(), 1);
    assertEquals(sw2.getPartition(0).getPartitionKey().getStartRow(), 0);
    assertEquals(sw2.getPartition(0).getPartitionKey().getEndRow(), 1);
    assertEquals(sw2.getPartition(0).getPartitionKey().getStartCol(), 0);
    assertEquals(sw2.getPartition(0).getPartitionKey().getEndCol(), 50000);
    assertEquals(sw2.getPartition(0).getPartitionKey().getMatrixId(), w2Id);
    assertEquals(sw2.getPartition(0).getPartitionKey().getPartitionId(), 0);
    assertEquals(sw2.getPartition(1).getPartitionKey().getStartRow(), 0);
    assertEquals(sw2.getPartition(1).getPartitionKey().getEndRow(), 1);
    assertEquals(sw2.getPartition(1).getPartitionKey().getStartCol(), 50000);
    assertEquals(sw2.getPartition(1).getPartitionKey().getEndCol(), 100000);
    assertEquals(sw2.getPartition(1).getPartitionKey().getMatrixId(), w2Id);
    assertEquals(sw2.getPartition(1).getPartitionKey().getPartitionId(), 1);
}

Example 5 with MatrixStorageManager

use of com.tencent.angel.ps.storage.MatrixStorageManager in project angel by Tencent.

the class GetPartFunc method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    if (partParam instanceof PartitionGetRowsParam) {
        PartitionGetRowsParam param = (PartitionGetRowsParam) partParam;
        PartitionKey pkey = param.getPartKey();
        pkey = psContext.getMatrixMetaManager().getMatrixMeta(pkey.getMatrixId()).getPartitionMeta(pkey.getPartitionId()).getPartitionKey();
        int ws = pkey.getStartRow();
        int es = pkey.getEndRow();
        List<Integer> reqRows = param.getRowIndexes();
        MatrixStorageManager manager = psContext.getMatrixStorageManager();
        List<ServerRow> rows = new ArrayList<>();
        for (int w : reqRows) rows.add(manager.getRow(pkey, w));
        PartCSRResult csr = new PartCSRResult(rows);
        return csr;
    } else {
        return null;
    }
}