use of com.tencent.angel.psagent.matrix.storage.MatrixStorage in project angel by Tencent.
the class ConsistencyController method findRowsInStorage.
private void findRowsInStorage(TaskContext taskContext, GetRowsResult result, RowIndex rowIndexes, int stalenessClock) throws InterruptedException {
MatrixStorage storage = PSAgentContext.get().getMatrixStorageManager().getMatrixStoage(rowIndexes.getMatrixId());
for (int rowIndex : rowIndexes.getRowIds()) {
TVector processRow = storage.getRow(rowIndex);
if (processRow != null && (taskContext.getPSMatrixClock(rowIndexes.getMatrixId()) <= processRow.getClock()) && (processRow.getClock() >= stalenessClock)) {
result.put(processRow);
if (result.getRowsNumber() == rowIndexes.getRowsNumber()) {
rowIndexes.clearFilted();
result.fetchOver();
return;
}
rowIndexes.filted(rowIndex);
}
}
}
use of com.tencent.angel.psagent.matrix.storage.MatrixStorage in project angel by Tencent.
the class CompSparseIntMatrixOpLog method flushToLocalStorage.
/**
* Flush the update in cache to local matrix storage
*/
public void flushToLocalStorage() {
MatrixStorage storage = PSAgentContext.get().getMatrixStorageManager().getMatrixStoage(matrixId);
MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
int row = matrixMeta.getRowNum();
TVector deltaVector = null;
TVector vector = null;
ReentrantReadWriteLock globalStorageLock = storage.getLock();
try {
globalStorageLock.writeLock().lock();
for (int rowIndex = 0; rowIndex < row; rowIndex++) {
deltaVector = getRow(rowIndex);
vector = storage.getRow(rowIndex);
if (deltaVector == null || vector == null)
continue;
vector.plusBy(deltaVector, 1.0 / PSAgentContext.get().getTotalTaskNum());
}
} finally {
globalStorageLock.writeLock().unlock();
}
}
use of com.tencent.angel.psagent.matrix.storage.MatrixStorage in project angel by Tencent.
the class MatrixClientAdapter method getRow.
/**
* Get a matrix row from parameter servers
*
* @param matrixId matrix id
* @param rowIndex row index
* @param clock clock value
* @return TVector matrix row
* @throws ExecutionException exception thrown when attempting to retrieve the result of a task
* that aborted by throwing an exception
* @throws InterruptedException interrupted while wait the result
*/
public TVector getRow(int matrixId, int rowIndex, int clock) throws InterruptedException, ExecutionException {
LOG.debug("start to getRow request, matrix=" + matrixId + ", rowIndex=" + rowIndex + ", clock=" + clock);
long startTs = System.currentTimeMillis();
// Wait until the clock value of this row is greater than or equal to the value
PSAgentContext.get().getConsistencyController().waitForClock(matrixId, rowIndex, clock);
LOG.debug("getRow wait clock time=" + (System.currentTimeMillis() - startTs));
startTs = System.currentTimeMillis();
// Get partitions for this row
List<PartitionKey> partList = PSAgentContext.get().getMatrixMetaManager().getPartitions(matrixId, rowIndex);
GetRowRequest request = new GetRowRequest(matrixId, rowIndex, clock);
MatrixMeta meta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
GetRowPipelineCache responseCache = (GetRowPipelineCache) requestToResponseMap.get(request);
if (responseCache == null) {
responseCache = new GetRowPipelineCache(partList.size(), meta.getRowType());
GetRowPipelineCache oldCache = (GetRowPipelineCache) requestToResponseMap.putIfAbsent(request, responseCache);
if (oldCache != null) {
responseCache = oldCache;
}
}
// First get this row from matrix storage
MatrixStorage matrixStorage = PSAgentContext.get().getMatrixStorageManager().getMatrixStoage(matrixId);
try {
responseCache.getDistinctLock().lock();
// If the row exists in the matrix storage and the clock value meets the requirements, just
// return
TVector row = matrixStorage.getRow(rowIndex);
if (row != null && row.getClock() >= clock) {
return row;
}
// Get row splits of this row from the matrix cache first
MatricesCache matricesCache = PSAgentContext.get().getMatricesCache();
MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
int size = partList.size();
for (int i = 0; i < size; i++) {
ServerRow rowSplit = matricesCache.getRowSplit(matrixId, partList.get(i), rowIndex);
if (rowSplit != null && rowSplit.getClock() >= clock) {
responseCache.addRowSplit(rowSplit);
} else {
// If the row split does not exist in cache, get it from parameter server
responseCache.addRowSplit(matrixClient.getRowSplit(partList.get(i), rowIndex, clock));
}
}
// Wait the final result
row = responseCache.getMergedResult().get();
LOG.debug("get row use time=" + (System.currentTimeMillis() - startTs));
// Put it to the matrix cache
matrixStorage.addRow(rowIndex, row);
return row;
} finally {
responseCache.getDistinctLock().unlock();
requestToResponseMap.remove(request);
}
}
Aggregations