Examples with DenseDoubleVector com.tencent.angel.ml.math.vector.DenseDoubleVector used on opensource projects

Example 1 with DenseDoubleVector

use of com.tencent.angel.ml.math.vector.DenseDoubleVector in project angel by Tencent.

the class TransportTest method testGetFlowDenseDoubleMatrix.

@Test
public void testGetFlowDenseDoubleMatrix() throws Exception {
    try {
        Worker worker = LocalClusterContext.get().getWorker(worker0Attempt0Id).getWorker();
        MatrixClient mat = worker.getPSAgent().getMatrixClient("dense_double_mat_1", 0);
        double[][] data = new double[ddRow][ddCol];
        DenseDoubleMatrix expect = new DenseDoubleMatrix(ddRow, ddCol, data);
        RowIndex rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        GetRowsResult result = mat.getRowsFlow(rowIndex, ddRow / 2);
        TVector row;
        while ((row = result.take()) != null) {
            LOG.info("===========get row index=" + row.getRowId());
            assertArrayEquals(((DenseDoubleVector) expect.getRow(row.getRowId())).getValues(), ((DenseDoubleVector) row).getValues(), 0.0);
        }
        Random rand = new Random(System.currentTimeMillis());
        for (int rowId = 0; rowId < ddRow; rowId++) {
            DenseDoubleVector update = new DenseDoubleVector(ddCol);
            for (int j = 0; j < ddCol; j += 3) update.set(j, rand.nextDouble());
            mat.increment(rowId, update);
            expect.getRow(rowId).plusBy(update);
        }
        mat.clock().get();
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while ((row = result.take()) != null) {
            assertArrayEquals(((DenseDoubleVector) expect.getRow(row.getRowId())).getValues(), ((DenseDoubleVector) row).getValues(), 0.0);
        }
        rowIndex = new RowIndex();
        for (int i = 0; i < ddRow; i++) rowIndex.addRowId(i);
        result = mat.getRowsFlow(rowIndex, 2);
        while (true) {
            row = result.poll();
            if (result.isFetchOver() && row == null)
                break;
            if (row == null)
                continue;
            assertArrayEquals(((DenseDoubleVector) expect.getRow(row.getRowId())).getValues(), ((DenseDoubleVector) row).getValues(), 0.0);
        }
    } catch (Exception x) {
        LOG.error("run testGetFlowDenseDoubleMatrix failed ", x);
        throw x;
    }
}

Example 2 with DenseDoubleVector

use of com.tencent.angel.ml.math.vector.DenseDoubleVector in project angel by Tencent.

the class LongKeyTestTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient client = taskContext.getMatrix("longkey_test");
        while (taskContext.getEpoch() < 100) {
            long startTs = System.currentTimeMillis();
            TVector row = client.getRow(0);
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " pull use time=" + (System.currentTimeMillis() - startTs) + ", sum=" + ((CompSparseLongKeyDoubleVector) row).sum());
            startTs = System.currentTimeMillis();
            CompSparseLongKeyDoubleVector deltaV = new CompSparseLongKeyDoubleVector(client.getMatrixId(), 0, 2100000000, 110000000);
            SparseLongKeyDoubleVector deltaV1 = new SparseLongKeyDoubleVector(2100000000, 150000000);
            DenseDoubleVector deltaV2 = new DenseDoubleVector(110000000);
            for (int i = 0; i < 2100000000; i += 20) {
                deltaV.set(i, 1.0);
                deltaV1.set(i, 1.0);
            }
            for (int i = 0; i < 110000000; i++) {
                deltaV2.set(i, 1.0);
            }
            startTs = System.currentTimeMillis();
            int tryNum = 100;
            while (tryNum-- > 0) {
                deltaV.timesBy(2.0);
            }
            LOG.info("combine times use time " + (System.currentTimeMillis() - startTs));
            startTs = System.currentTimeMillis();
            tryNum = 100;
            while (tryNum-- > 0) {
                deltaV1.timesBy(2.0);
            }
            LOG.info("single times use time " + (System.currentTimeMillis() - startTs));
            startTs = System.currentTimeMillis();
            tryNum = 100;
            while (tryNum-- > 0) {
                deltaV2.timesBy(2.0);
            }
            LOG.info("dense times use time " + (System.currentTimeMillis() - startTs));
            deltaV.setMatrixId(client.getMatrixId());
            deltaV.setRowId(0);
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " train use time=" + (System.currentTimeMillis() - startTs));
            startTs = System.currentTimeMillis();
            client.increment(deltaV);
            client.clock().get();
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " flush use time=" + (System.currentTimeMillis() - startTs));
            taskContext.incEpoch();
        }
    } catch (Throwable x) {
        throw new AngelException("run task failed ", x);
    }
}

Example 3 with DenseDoubleVector

use of com.tencent.angel.ml.math.vector.DenseDoubleVector in project angel by Tencent.

the class PSFTestTask method run.

@Override
public void run(TaskContext taskContext) throws AngelException {
    try {
        MatrixClient client = taskContext.getMatrix("psf_test");
        Pull func = new Pull(client.getMatrixId(), 0);
        Pull func1 = new Pull(client.getMatrixId(), 1);
        while (taskContext.getEpoch() < 100) {
            taskContext.globalSync(client.getMatrixId());
            long startTs = System.currentTimeMillis();
            TVector row = ((GetRowResult) client.get(func)).getRow();
            TVector row1 = ((GetRowResult) client.get(func1)).getRow();
            LOG.info("Task " + taskContext.getTaskId() + " in iteration " + taskContext.getEpoch() + " pull use time=" + (System.currentTimeMillis() - startTs) + ", sum of row 0=" + sum((DenseDoubleVector) row) + " sum of row 1=" + sum((DenseDoubleVector) row1));
            double[] delta = new double[10000000];
            for (int i = 0; i < 10000000; i++) {
                delta[i] = 1.0;
            }
            DenseDoubleVector deltaV = new DenseDoubleVector(10000000, delta);
            deltaV.setMatrixId(client.getMatrixId());
            deltaV.setRowId(0);
            double[] delta1 = new double[10000000];
            for (int i = 0; i < 10000000; i++) {
                delta1[i] = 2.0;
            }
            DenseDoubleVector deltaV1 = new DenseDoubleVector(10000000, delta1);
            deltaV1.setMatrixId(client.getMatrixId());
            deltaV1.setRowId(1);
            client.increment(deltaV);
            client.increment(deltaV1);
            client.clock().get();
            taskContext.incEpoch();
        }
    } catch (Throwable x) {
        throw new AngelException("run task failed ", x);
    }
}

Example 4 with DenseDoubleVector

use of com.tencent.angel.ml.math.vector.DenseDoubleVector in project angel by Tencent.

the class GradHistThread method run.

@Override
public void run() {
    LOG.debug(String.format("Run active node[%d]", this.nid));
    // 1. name of this node's grad histogram on PS
    String histParaName = this.controller.param.gradHistNamePrefix + nid;
    // 2. build the grad histogram of this node
    GradHistHelper histMaker = new GradHistHelper(this.controller, this.nid);
    DenseDoubleVector histogram = histMaker.buildHistogram(insStart, insEnd);
    int bytesPerItem = this.controller.taskContext.getConf().getInt(MLConf.ML_COMPRESS_BYTES(), MLConf.DEFAULT_ML_COMPRESS_BYTES());
    if (bytesPerItem < 1 || bytesPerItem > 8) {
        LOG.info("Invalid compress configuration: " + bytesPerItem + ", it should be [1,8].");
        bytesPerItem = MLConf.DEFAULT_ML_COMPRESS_BYTES();
    }
    // 3. push the histograms to PS
    try {
        if (bytesPerItem == 8) {
            this.model.increment(0, histogram);
        } else {
            CompressUpdateFunc func = new CompressUpdateFunc(this.model.getMatrixId(), 0, histogram, bytesPerItem * 8);
            this.model.update(func);
        }
    } catch (Exception e) {
        LOG.error(histParaName + " increment failed, ", e);
    }
    // 4. reset thread stats to finished
    this.controller.activeNodeStat[this.nid]--;
    LOG.debug(String.format("Active node[%d] finish", this.nid));
}

Example 5 with DenseDoubleVector

use of com.tencent.angel.ml.math.vector.DenseDoubleVector in project angel by Tencent.

the class L1LogLossTest method testPredict.

@Test
public void testPredict() throws Exception {
    double[] data1 = { 1.0, 2.0, 3.0, 4.0 };
    DenseDoubleVector denseDoubleVector1 = new DenseDoubleVector(4, data1);
    double[] data2 = { 1.0, 2.0, 3.0, 4.0 };
    DenseDoubleVector denseDoubleVector2 = new DenseDoubleVector(4, data2);
    double test = l1LogLoss.predict(denseDoubleVector1, denseDoubleVector2);
    double dot = 0.0;
    for (int i = 0; i < data1.length; i++) dot += data1[i] * data2[i];
    assertEquals(dot, test, 0.00);
}