Examples with MatrixMeta com.tencent.angel.ml.matrix.MatrixMeta used on opensource projects

Example 21 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class PSAgent method releaseMatrices.

/**
 * Release a batch of matrices
 *
 * @param matrixIds matrix ids
 * @throws AngelException exception come from master
 */
public void releaseMatrices(List<Integer> matrixIds) throws AngelException {
    int size = matrixIds.size();
    List<String> matrixNames = new ArrayList<>(size);
    for (int i = 0; i < size; i++) {
        MatrixMeta meta = matrixMetaManager.getMatrixMeta(matrixIds.get(i));
        if (meta == null) {
            continue;
        }
        matrixNames.add(meta.getName());
    }
    releaseMatricesUseName(matrixNames);
}

Example 22 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class PSAgent method initAndStart.

public void initAndStart() throws Exception {
    // Init control connection manager
    controlConnectManager = TConnectionManager.getConnection(conf);
    // Get ps locations from master and put them to the location cache.
    locationManager = new PSAgentLocationManager(PSAgentContext.get());
    locationManager.setMasterLocation(masterLocation);
    // Build and initialize rpc client to master
    masterClient = new MasterClient();
    masterClient.init();
    // Get psagent id
    id = masterClient.getPSAgentId();
    // Build PS control rpc client manager
    psControlClientManager = new PSControlClientManager();
    // Build local location
    String localIp = NetUtils.getRealLocalIP();
    int port = NetUtils.chooseAListenPort(conf);
    location = new Location(localIp, port);
    register();
    // Initialize matrix meta information
    // clockCache = new ClockCache();
    List<MatrixMeta> matrixMetas = masterClient.getMatrices();
    LOG.info("PSAgent get matrices from master," + matrixMetas.size());
    this.matrixMetaManager = new PSAgentMatrixMetaManager();
    matrixMetaManager.addMatrices(matrixMetas);
    Map<ParameterServerId, Location> psIdToLocMap = masterClient.getPSLocations();
    List<ParameterServerId> psIds = new ArrayList<>(psIdToLocMap.keySet());
    Collections.sort(psIds, new Comparator<ParameterServerId>() {

        @Override
        public int compare(ParameterServerId s1, ParameterServerId s2) {
            return s1.getIndex() - s2.getIndex();
        }
    });
    int size = psIds.size();
    locationManager.setPsIds(psIds.toArray(new ParameterServerId[0]));
    for (int i = 0; i < size; i++) {
        if (psIdToLocMap.containsKey(psIds.get(i))) {
            locationManager.setPsLocation(psIds.get(i), psIdToLocMap.get(psIds.get(i)));
        }
    }
    matrixTransClient = new MatrixTransportClient();
    userRequestAdapter = new UserRequestAdapter();
    if (runningMode == RunningMode.ANGEL_PS_WORKER) {
        // opLogCache = new MatrixOpLogCache();
        matrixStorageManager = new MatrixStorageManager();
    // int staleness = conf.getInt(AngelConf.ANGEL_STALENESS, AngelConf.DEFAULT_ANGEL_STALENESS);
    // consistencyController = new ConsistencyController(staleness);
    // consistencyController.init();
    }
    psAgentInitFinishedFlag.set(true);
    // Start all services
    matrixTransClient.start();
    userRequestAdapter.start();
    if (runningMode == RunningMode.ANGEL_PS_WORKER) {
    // clockCache.start();
    // opLogCache.start();
    }
}

Example 23 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class UserRequestAdapter method get.

private FutureResult<Vector[]> get(IndexGetRowsRequest request) {
    // Only support column-partitioned matrix now!!
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(request.getMatrixId());
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    for (PartitionKey part : matrixParts) {
        if (part.getStartRow() != 0 || part.getEndRow() != matrixMeta.getRowNum()) {
            throw new UnsupportedOperationException("Get rows by indices only support column-partitioned matrix now");
        }
    }
    // Split the user request to partition requests
    FutureResult<Vector[]> result = new FutureResult<>();
    long startTs = System.currentTimeMillis();
    KeyPart[] splits;
    if (request instanceof IntIndexGetRowsRequest) {
        splits = RouterUtils.split(matrixMeta, -1, ((IntIndexGetRowsRequest) request).getIndices());
    } else if (request instanceof LongIndexGetRowsRequest) {
        splits = RouterUtils.split(matrixMeta, -1, ((LongIndexGetRowsRequest) request).getIndices());
    } else {
        throw new UnsupportedOperationException("Unsupport index request type " + request.getClass().toString());
    }
    LOG.info("Get by indices split use time: " + (System.currentTimeMillis() - startTs));
    assert matrixParts.length == splits.length;
    // filter empty partition requests
    int needRequestPartNum = noEmptyPartNum(splits);
    if (needRequestPartNum == 0) {
        result.set(null);
        return result;
    }
    // Create partition results cache
    ResponseCache cache = new MapResponseCache(needRequestPartNum);
    int requestId = request.getRequestId();
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    requests.put(requestId, request);
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (int i = 0; i < splits.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            sendIndexGetRowsRequest(matrixClient, requestId, request.getMatrixId(), request.getRowIds(), matrixParts[i].getPartitionId(), splits[i], request.getFunc());
        }
    }
    return result;
}

Example 24 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class UserRequestAdapter method checkParams.

private void checkParams(int matrixId, int rowId) {
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    if (matrixMeta == null) {
        throw new AngelException("can not find matrix " + matrixId);
    }
    int rowNum = matrixMeta.getRowNum();
    if (rowId < 0 || rowId >= rowNum) {
        throw new AngelException("not valid row id, row id is in range[0," + rowNum + ")");
    }
}

Example 25 with MatrixMeta

use of com.tencent.angel.ml.matrix.MatrixMeta in project angel by Tencent.

the class GeneralInitParam method split.

@Override
public List<PartitionUpdateParam> split() {
    MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = meta.getPartitionKeys();
    KeyValuePart[] splits = RouterUtils.split(meta, 0, nodeIds, features);
    assert parts.length == splits.length;
    List<PartitionUpdateParam> partParams = new ArrayList<>(parts.length);
    for (int i = 0; i < parts.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            partParams.add(new GeneralPartUpdateParam(matrixId, parts[i], splits[i]));
        }
    }
    return partParams;
}