Examples with MatrixTransportClient com.tencent.angel.psagent.matrix.transport.MatrixTransportClient used on opensource projects

Example 1 with MatrixTransportClient

use of com.tencent.angel.psagent.matrix.transport.MatrixTransportClient in project angel by Tencent.

the class MatrixClientAdapter method update.

/**
 * Update matrix use a udf.
 *
 * @param updateFunc update udf function
 * @return Future<VoidResult> update future result
 */
public Future<VoidResult> update(UpdateFunc updateFunc) {
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    UpdateParam param = updateFunc.getParam();
    List<PartitionUpdateParam> partParams = param.split();
    int size = partParams.size();
    UpdaterRequest request = new UpdaterRequest(param);
    UpdaterResponseCache cache = new UpdaterResponseCache(size);
    for (int i = 0; i < size; i++) {
        cache.addResult(matrixClient.update(updateFunc, partParams.get(i)));
    }
    requestToResponseMap.put(request, cache);
    return cache.getMergedResult();
}

Example 2 with MatrixTransportClient

use of com.tencent.angel.psagent.matrix.transport.MatrixTransportClient in project angel by Tencent.

the class MatrixClientAdapter method dispatchGetRows.

/**
 * Split the rowIndex to batches to generate rpc dispatcher items
 *
 * @param rowIndex rowIds needed to been requested from PS
 */
private void dispatchGetRows(RowIndex rowIndex, int rpcBatchSize, int clock) {
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    // Get the partition to sub-row splits map:use to storage the rows stored in a matrix partition
    Map<PartitionKey, List<RowIndex>> partToRowIndexMap = PSAgentContext.get().getMatrixMetaManager().getPartitionToRowIndexMap(rowIndex, rpcBatchSize);
    List<RowIndex> rowIds;
    int size;
    // Generate dispatch items and add them to the corresponding queues
    int totalRequestNumber = 0;
    for (Entry<PartitionKey, List<RowIndex>> entry : partToRowIndexMap.entrySet()) {
        totalRequestNumber += entry.getValue().size();
    }
    GetRowsFlowRequest request = new GetRowsFlowRequest(rowIndex, clock);
    // Filter the rowIds which are fetching now
    ReentrantLock lock = getLock(rowIndex.getMatrixId());
    Int2IntOpenHashMap rowIndexToPartSizeMap;
    try {
        lock.lock();
        rowIndexToPartSizeMap = matrixToRowSplitSizeCache.get(rowIndex.getMatrixId());
    } finally {
        lock.unlock();
    }
    GetRowsFlowCache cache = new GetRowsFlowCache(totalRequestNumber, rowIndex.getMatrixId(), rowIndexToPartSizeMap);
    for (Entry<PartitionKey, List<RowIndex>> entry : partToRowIndexMap.entrySet()) {
        totalRequestNumber += entry.getValue().size();
        rowIds = entry.getValue();
        size = rowIds.size();
        for (int i = 0; i < size; i++) {
            cache.addResult(matrixClient.getRowsSplit(entry.getKey(), rowIndexToList(rowIds.get(i)), clock));
        }
    }
    requestToResponseMap.put(request, cache);
}

Example 3 with MatrixTransportClient

use of com.tencent.angel.psagent.matrix.transport.MatrixTransportClient in project angel by Tencent.

the class PSAgent method initAndStart.

public void initAndStart() throws Exception {
    // Init control connection manager
    controlConnectManager = TConnectionManager.getConnection(conf);
    // Get ps locations from master and put them to the location cache.
    locationManager = new PSAgentLocationManager(PSAgentContext.get());
    locationManager.setMasterLocation(masterLocation);
    // Build and initialize rpc client to master
    masterClient = new MasterClient();
    masterClient.init();
    // Get psagent id
    id = masterClient.getPSAgentId();
    // Build PS control rpc client manager
    psControlClientManager = new PSControlClientManager();
    // Build local location
    String localIp = NetUtils.getRealLocalIP();
    int port = NetUtils.chooseAListenPort(conf);
    location = new Location(localIp, port);
    register();
    // Initialize matrix meta information
    // clockCache = new ClockCache();
    List<MatrixMeta> matrixMetas = masterClient.getMatrices();
    LOG.info("PSAgent get matrices from master," + matrixMetas.size());
    this.matrixMetaManager = new PSAgentMatrixMetaManager();
    matrixMetaManager.addMatrices(matrixMetas);
    Map<ParameterServerId, Location> psIdToLocMap = masterClient.getPSLocations();
    List<ParameterServerId> psIds = new ArrayList<>(psIdToLocMap.keySet());
    Collections.sort(psIds, new Comparator<ParameterServerId>() {

        @Override
        public int compare(ParameterServerId s1, ParameterServerId s2) {
            return s1.getIndex() - s2.getIndex();
        }
    });
    int size = psIds.size();
    locationManager.setPsIds(psIds.toArray(new ParameterServerId[0]));
    for (int i = 0; i < size; i++) {
        if (psIdToLocMap.containsKey(psIds.get(i))) {
            locationManager.setPsLocation(psIds.get(i), psIdToLocMap.get(psIds.get(i)));
        }
    }
    matrixTransClient = new MatrixTransportClient();
    userRequestAdapter = new UserRequestAdapter();
    if (runningMode == RunningMode.ANGEL_PS_WORKER) {
        // opLogCache = new MatrixOpLogCache();
        matrixStorageManager = new MatrixStorageManager();
    // int staleness = conf.getInt(AngelConf.ANGEL_STALENESS, AngelConf.DEFAULT_ANGEL_STALENESS);
    // consistencyController = new ConsistencyController(staleness);
    // consistencyController.init();
    }
    psAgentInitFinishedFlag.set(true);
    // Start all services
    matrixTransClient.start();
    userRequestAdapter.start();
    if (runningMode == RunningMode.ANGEL_PS_WORKER) {
    // clockCache.start();
    // opLogCache.start();
    }
}

Example 4 with MatrixTransportClient

use of com.tencent.angel.psagent.matrix.transport.MatrixTransportClient in project angel by Tencent.

the class UserRequestAdapter method update.

public Future<VoidResult> update(int matrixId, int rowId, Vector delta, UpdateOp op) {
    checkParams(matrixId, rowId);
    delta.setMatrixId(matrixId);
    delta.setRowId(rowId);
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    CompStreamKeyValuePart[] splits = RouterUtils.splitStream(matrixMeta, delta);
    FutureResult<VoidResult> result = new FutureResult<>();
    int needRequestPartNum = noEmptyPartNum(splits);
    if (needRequestPartNum == 0) {
        result.set(null);
        return result;
    }
    UpdateRowRequest request = new UpdateRowRequest(matrixId, rowId, op);
    ResponseCache cache = new MapResponseCache(needRequestPartNum);
    int requestId = request.getRequestId();
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    requests.put(requestId, request);
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (int i = 0; i < splits.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            sendUpdateRequest(matrixClient, requestId, matrixId, parts[i].getPartitionId(), splits[i], op);
        }
    }
    return result;
}

Example 5 with MatrixTransportClient

use of com.tencent.angel.psagent.matrix.transport.MatrixTransportClient in project angel by Tencent.

the class UserRequestAdapter method update.

/**
 * Update matrix use a udf.
 *
 * @param updateFunc update udf function
 * @return Future<VoidResult> update future result
 */
public Future<VoidResult> update(UpdateFunc updateFunc) {
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    UpdateParam param = updateFunc.getParam();
    // Split the param use matrix partitions
    List<PartitionUpdateParam> partParams = param.split();
    int size = partParams.size();
    UpdatePSFRequest request = new UpdatePSFRequest(updateFunc);
    ResponseCache cache = new MapResponseCache(size);
    FutureResult<VoidResult> result = new FutureResult<>();
    int requestId = request.getRequestId();
    requests.put(requestId, request);
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    // Send request to PSS
    for (int i = 0; i < size; i++) {
        sendUpdateUDFRequest(matrixClient, requestId, partParams.get(i).getMatrixId(), partParams.get(i).getPartKey().getPartitionId(), updateFunc, partParams.get(i));
    }
    return result;
}