Examples with PartitionKey com.tencent.angel.PartitionKey used on opensource projects

Example 31 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ValuesCombineUtils method mergeSparseDoubleVector.

public static SparseLongKeyDoubleVector mergeSparseDoubleVector(LongIndexGetParam param, List<PartitionGetResult> partResults) {
    SparseLongKeyDoubleVector vector = new SparseLongKeyDoubleVector(PSAgentContext.get().getMatrixMetaManager().getMatrixMeta(param.getMatrixId()).getColNum(), param.size());
    for (PartitionGetResult part : partResults) {
        PartitionKey partKey = ((LongIndexGetResult) part).partKey;
        long[] indexes = param.getPartKeyToIndexesMap().get(partKey);
        double[] values = ((LongIndexGetResult) part).getValues();
        for (int i = 0; i < indexes.length; i++) {
            vector.set(indexes[i], values[i]);
        }
    }
    vector.setMatrixId(param.getMatrixId());
    vector.setRowId(param.getRowId());
    return vector;
}

Example 32 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class GetClocksResponse method deserialize.

@Override
public void deserialize(ByteBuf buf) {
    super.deserialize(buf);
    if (buf.readableBytes() != 0) {
        int size = buf.readInt();
        clocks = new HashMap<>(size);
        for (int i = 0; i < size; i++) {
            PartitionKey partKey = new PartitionKey();
            partKey.deserialize(buf);
            clocks.put(partKey, buf.readInt());
        }
    }
}

Example 33 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class PartitionRequest method deserialize.

@Override
public void deserialize(ByteBuf buf) {
    super.deserialize(buf);
    comeFromPs = buf.readBoolean();
    clock = buf.readInt();
    partKey = new PartitionKey();
    partKey.deserialize(buf);
// if(comeFromPs) {
// psId = new ParameterServerId(buf.readInt());
// int size = buf.readInt();
// byte[] data = new byte[size];
// buf.readBytes(data);
// location = new Location(new String(data), buf.readInt());
// }
}

Example 34 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class TaskManager method combineUpdateIndex.

/**
 * Combine update index.
 */
@SuppressWarnings("rawtypes")
public void combineUpdateIndex() {
    IntOpenHashSet indexSet = null;
    MatrixMeta meta = null;
    for (Entry<TaskId, Task> entry : runningTask.entrySet()) {
        LabeledUpdateIndexBaseTask task = (LabeledUpdateIndexBaseTask) entry.getValue().getUserTask();
        IntOpenHashSet taskIndexSet = task.getIndexSet();
        if (taskIndexSet != null) {
            if (indexSet == null) {
                indexSet = taskIndexSet;
                meta = task.getMatrixMeta();
            } else {
                indexSet.addAll(taskIndexSet);
                task.setIndexSet(null);
            }
        }
    }
    if (indexSet != null && meta != null) {
        int size = indexSet.size();
        int[] indexArray = new int[size];
        int index = 0;
        IntIterator iter = indexSet.iterator();
        while (iter.hasNext()) {
            indexArray[index++] = iter.nextInt();
        }
        Arrays.sort(indexArray);
        List<PartitionKey> partKeyList = WorkerContext.get().getPSAgent().getMatrixMetaManager().getPartitions(meta.getId());
        Collections.sort(partKeyList);
        int partNum = partKeyList.size();
        int lastPos = 0;
        for (int i = 0; i < partNum; i++) {
            PartitionKey partKey = partKeyList.get(i);
            long endCol = partKey.getEndCol();
            for (int j = lastPos; j < size; j++) {
                if (indexArray[j] >= endCol) {
                    lastPos = j;
                    break;
                }
            }
        }
    // Bitmap bitmap = new Bitmap();
    // int max = indexArray[size - 1];
    // byte [] bitIndexArray = new byte[max / 8 + 1];
    // for(int i = 0; i < size; i++){
    // int bitIndex = indexArray[i] >> 3;
    // int bitOffset = indexArray[i] - (bitIndex << 3);
    // switch(bitOffset){
    // case 0:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x01);break;
    // case 1:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x02);break;
    // case 2:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x04);break;
    // case 3:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x08);break;
    // case 4:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x10);break;
    // case 5:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x20);break;
    // case 6:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x40);break;
    // case 7:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x80);break;
    // }
    // }
    }
}

Example 35 with PartitionKey

use of com.tencent.angel.PartitionKey in project angel by Tencent.

the class ServerPartitionTest method testWriteTo.

@Test
public void testWriteTo() throws Exception {
    // set basic configuration keys
    conf = new Configuration();
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setBoolean(AngelConf.ANGEL_JOB_OUTPUT_PATH_DELETEONEXIST, true);
    conf.set(AngelConf.ANGEL_TASK_USER_TASKCLASS, DummyTask.class.getName());
    // use local deploy mode and dummy dataspliter
    conf.set(AngelConf.ANGEL_DEPLOY_MODE, "LOCAL");
    conf.setBoolean(AngelConf.ANGEL_AM_USE_DUMMY_DATASPLITER, true);
    conf.set(AngelConf.ANGEL_INPUTFORMAT_CLASS, CombineTextInputFormat.class.getName());
    conf.set(AngelConf.ANGEL_SAVE_MODEL_PATH, LOCAL_FS + TMP_PATH + "/out");
    conf.set(AngelConf.ANGEL_TRAIN_DATA_PATH, LOCAL_FS + TMP_PATH + "/in");
    conf.set(AngelConf.ANGEL_LOG_PATH, LOCAL_FS + TMP_PATH + "/log");
    conf.setInt(AngelConf.ANGEL_WORKERGROUP_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_PS_NUMBER, 1);
    conf.setInt(AngelConf.ANGEL_WORKER_TASK_NUMBER, 2);
    // get a angel client
    angelClient = AngelClientFactory.get(conf);
    // add matrix
    MatrixContext mMatrix = new MatrixContext();
    mMatrix.setName("w1");
    mMatrix.setRowNum(1);
    mMatrix.setColNum(100000);
    mMatrix.setMaxRowNumInBlock(1);
    mMatrix.setMaxColNumInBlock(50000);
    mMatrix.setRowType(RowType.T_INT_DENSE);
    mMatrix.set(MatrixConf.MATRIX_OPLOG_ENABLEFILTER, "false");
    mMatrix.set(MatrixConf.MATRIX_HOGWILD, "true");
    mMatrix.set(MatrixConf.MATRIX_AVERAGE, "false");
    mMatrix.set(MatrixConf.MATRIX_OPLOG_TYPE, "DENSE_INT");
    angelClient.addMatrix(mMatrix);
    angelClient.startPSServer();
    angelClient.runTask(DummyTask.class);
    Thread.sleep(5000);
    group0Id = new WorkerGroupId(0);
    worker0Id = new WorkerId(group0Id, 0);
    worker0Attempt0Id = new WorkerAttemptId(worker0Id, 0);
    task0Id = new TaskId(0);
    task1Id = new TaskId(1);
    psId = new ParameterServerId(0);
    psAttempt0Id = new PSAttemptId(psId, 0);
    DataOutputStream out = new DataOutputStream(new FileOutputStream("data"));
    ByteBuf buf = Unpooled.buffer(16);
    buf.writeDouble(0.00);
    buf.writeDouble(1.00);
    buf.writeDouble(-1.00);
    buf.writeDouble(-2.00);
    buf.writeDouble(-5.00);
    buf.writeDouble(-6.00);
    buf.writeDouble(-7.00);
    buf.writeDouble(-8.00);
    serverPartition.getRow(6).update(RowType.T_DOUBLE_DENSE, buf, 8);
    serverPartition.save(out);
    out.close();
    DataInputStream in = new DataInputStream(new FileInputStream("data"));
    PartitionKey partitionKeyNew = new PartitionKey(2, 1, 1, 2, 8, 10);
    ServerPartition serverPartitionNew = new ServerPartition(partitionKeyNew, RowType.T_DOUBLE_DENSE);
    serverPartitionNew.init();
    assertNotEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(), ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
    serverPartitionNew.load(in);
    in.close();
    assertEquals(((ServerDenseDoubleRow) serverPartition.getRow(6)).getData(), ((ServerDenseDoubleRow) serverPartitionNew.getRow(6)).getData());
    angelClient.stop();
}