Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 81 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class MatrixFormatImpl method save.

private void save(ServerMatrix matrix, Path matrixPath, FileSystem fs, List<Integer> partIds, PSMatrixSaveContext saveContext, int startPos, int endPos, PSMatrixFilesMeta dataFilesMeta) throws IOException {
    Path destFile = new Path(matrixPath, ModelFilesUtils.fileName(partIds.get(startPos)));
    Path tmpDestFile = HdfsUtil.toTmpPath(destFile);
    if (fs.exists(tmpDestFile)) {
        fs.delete(tmpDestFile, true);
    }
    FSDataOutputStream out = fs.create(tmpDestFile, true, conf.getInt(AngelConf.ANGEL_PS_IO_FILE_BUFFER_SIZE, AngelConf.DEFAULT_ANGEL_PS_IO_FILE_BUFFER_SIZE));
    long streamPos = 0;
    ServerPartition partition;
    for (int i = startPos; i < endPos; i++) {
        LOG.info("Write partition " + partIds.get(i) + " of matrix " + matrix.getName() + " to " + tmpDestFile);
        streamPos = out.getPos();
        partition = matrix.getPartition(partIds.get(i));
        PartitionKey partKey = partition.getPartitionKey();
        MatrixPartitionMeta partMeta;
        long startCol = 0;
        long endCol = 0;
        if (useRange) {
            startCol = partKey.getStartCol();
            endCol = partKey.getEndCol();
        }
        partMeta = new MatrixPartitionMeta(partKey.getPartitionId(), partKey.getStartRow(), partKey.getEndRow(), startCol, endCol, partition.getElemNum(), destFile.getName(), streamPos, 0);
        save(partition, partMeta, saveContext, out);
        partMeta.setLength(out.getPos() - streamPos);
        dataFilesMeta.addPartitionMeta(partIds.get(i), partMeta);
    }
    out.flush();
    out.close();
    HdfsUtil.rename(tmpDestFile, destFile, fs);
}

Example 82 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class HashRouterUtils method split.

/**
 * Split keys by matrix partition
 *
 * @param matrixMeta matrix meta data
 * @param vector Matrix vector
 * @return partition key to key partition map
 */
public static KeyValuePart[] split(MatrixMeta matrixMeta, Vector vector) {
    KeyHash hasher = HasherFactory.getHasher(matrixMeta.getRouterHash());
    PartitionKey[] matrixParts = matrixMeta.getPartitionKeys();
    KeyValuePart[] dataParts = new KeyValuePart[matrixParts.length];
    int estSize = (int) (vector.getSize() / matrixMeta.getPartitionNum());
    for (int i = 0; i < dataParts.length; i++) {
        dataParts[i] = generateDataPart(vector.getRowId(), vector.getType(), estSize);
    }
    switch(vector.getType()) {
        case T_DOUBLE_DENSE:
        case T_DOUBLE_SPARSE:
            {
                splitIntDoubleVector(hasher, matrixMeta, (IntDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_DENSE:
        case T_FLOAT_SPARSE:
            {
                splitIntFloatVector(hasher, matrixMeta, (IntFloatVector) vector, dataParts);
                break;
            }
        case T_INT_DENSE:
        case T_INT_SPARSE:
            {
                splitIntIntVector(hasher, matrixMeta, (IntIntVector) vector, dataParts);
                break;
            }
        case T_LONG_DENSE:
        case T_LONG_SPARSE:
            {
                splitIntLongVector(hasher, matrixMeta, (IntLongVector) vector, dataParts);
                break;
            }
        case T_DOUBLE_SPARSE_LONGKEY:
            {
                splitLongDoubleVector(hasher, matrixMeta, (LongDoubleVector) vector, dataParts);
                break;
            }
        case T_FLOAT_SPARSE_LONGKEY:
            {
                splitLongFloatVector(hasher, matrixMeta, (LongFloatVector) vector, dataParts);
                break;
            }
        case T_INT_SPARSE_LONGKEY:
            {
                splitLongIntVector(hasher, matrixMeta, (LongIntVector) vector, dataParts);
                break;
            }
        case T_LONG_SPARSE_LONGKEY:
            {
                splitLongLongVector(hasher, matrixMeta, (LongLongVector) vector, dataParts);
                break;
            }
        default:
            {
                throw new UnsupportedOperationException("Unsupport vector type " + vector.getType());
            }
    }
    for (int i = 0; i < dataParts.length; i++) {
        if (dataParts[i] != null) {
            dataParts[i].setRowId(vector.getRowId());
        }
    }
    return dataParts;
}

Example 83 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class PSAgentTest method testMatrixLocationManager.

@Test
public void testMatrixLocationManager() throws Exception {
    try {
        AngelApplicationMaster angelAppMaster = LocalClusterContext.get().getMaster().getAppMaster();
        assertTrue(angelAppMaster != null);
        AMTaskManager taskManager = angelAppMaster.getAppContext().getTaskManager();
        assertTrue(taskManager != null);
        WorkerManager workerManager = angelAppMaster.getAppContext().getWorkerManager();
        assertTrue(workerManager != null);
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        assertTrue(worker != null);
        PSAgent psAgent = worker.getPSAgent();
        assertTrue(psAgent != null);
        PSAgentMatrixMetaManager matrixPartitionRouter = psAgent.getMatrixMetaManager();
        PSAgentLocationManager locationCache = psAgent.getLocationManager();
        assertTrue(matrixPartitionRouter != null);
        // test ps location
        Location psLoc = locationCache.getPsLocation(psId);
        String ipRegex = "(2[5][0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})\\.(25[0-5]|2[0-4]\\d|1\\d{2}|\\d{1,2})";
        Pattern pattern = Pattern.compile(ipRegex);
        Matcher matcher = pattern.matcher(psLoc.getIp());
        assertTrue(matcher.matches());
        assertTrue(psLoc.getPort() >= 1 && psLoc.getPort() <= 65535);
        int matrix1Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w1").getId();
        int matrix2Id = LocalClusterContext.get().getMaster().getAppMaster().getAppContext().getMatrixMetaManager().getMatrix("w2").getId();
        // test partitions
        List<PartitionKey> partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id);
        assertEquals(partition1Keys.size(), 2);
        List<PartitionKey> partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id);
        assertEquals(partition2Keys.size(), 2);
        partition1Keys.clear();
        partition1Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
        assertEquals(partition1Keys.size(), 2);
        partition2Keys.clear();
        partition2Keys = matrixPartitionRouter.getPartitions(matrix1Id, 0);
        assertEquals(partition2Keys.size(), 2);
        int rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
        assertEquals(rowPartSize, 2);
        rowPartSize = matrixPartitionRouter.getRowPartitionSize(matrix1Id, 0);
        assertEquals(rowPartSize, 2);
    } catch (Exception x) {
        LOG.error("run testMatrixLocationManager failed ", x);
        throw x;
    }
}

Example 84 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ServerPartition method deserialize.

@Override
public void deserialize(ByteBuf buf) {
    // Deserialize the head
    partKey = new PartitionKey();
    partKey.deserialize(buf);
    rowType = RowType.valueOf(buf.readInt());
    // Deseralize the storage
    int size = buf.readInt();
    byte[] data = new byte[size];
    buf.readBytes(data);
    String storageClassName = new String(data);
    storage = ServerPartitionStorageFactory.getStorage(storageClassName);
    storage.deserialize(buf);
}

Example 85 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class UserRequestAdapter method update.

public Future<VoidResult> update(int matrixId, int[] rowIds, Vector[] rows, UpdateOp op) {
    assert rowIds.length == rows.length;
    checkParams(matrixId, rowIds);
    for (int i = 0; i < rowIds.length; i++) {
        rows[i].setRowId(rowIds[i]);
    }
    MatrixMeta matrixMeta = PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(matrixId);
    PartitionKey[] parts = matrixMeta.getPartitionKeys();
    CompStreamKeyValuePart[] splits = RouterUtils.splitStream(matrixMeta, rows);
    int needRequestPartNum = noEmptyPartNum(splits);
    FutureResult<VoidResult> result = new FutureResult<>();
    if (needRequestPartNum == 0) {
        result.set(null);
        return result;
    }
    UpdateRowsRequest request = new UpdateRowsRequest(matrixId, op);
    ResponseCache cache = new MapResponseCache(needRequestPartNum);
    int requestId = request.getRequestId();
    requestIdToResponseCache.put(requestId, cache);
    requestIdToResultMap.put(requestId, result);
    requests.put(requestId, request);
    MatrixTransportClient matrixClient = PSAgentContext.get().getMatrixTransportClient();
    for (int i = 0; i < splits.length; i++) {
        if (splits[i] != null && splits[i].size() > 0) {
            sendUpdateRequest(matrixClient, requestId, matrixId, parts[i].getPartitionId(), splits[i], op);
        }
    }
    return result;
}