Examples with Long2DoubleOpenHashMap it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap used on opensource projects

Example 31 with Long2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap in project angel by Tencent.

the class ModelLoader method loadSparseDoubleLongKeyRowFromPartition.

public static Long2DoubleOpenHashMap loadSparseDoubleLongKeyRowFromPartition(FSDataInputStream input, MatrixPartitionMeta partMeta, int rowId) throws IOException {
    // RowOffset rowOffset = partMeta.getRowMetas().get(rowId);
    // input.seek(rowOffset.getOffset());
    Preconditions.checkState(input.readInt() == rowId);
    int num = input.readInt();
    Long2DoubleOpenHashMap row = new Long2DoubleOpenHashMap();
    for (int j = 0; j < num; j++) {
        row.put(input.readLong(), input.readDouble());
    }
    return row;
}

Example 32 with Long2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap in project angel by Tencent.

the class ModelLoader method loadSparseDoubleLongKeyPartition.

private static void loadSparseDoubleLongKeyPartition(SparseDoubleLongKeyModel model, FSDataInputStream input, MatrixPartitionMeta partMeta) throws IOException {
    int rowNum = input.readInt();
    int rowId = 0;
    int nnz = 0;
    int totalNNZ = 0;
    Long2DoubleOpenHashMap row = null;
    for (int i = 0; i < rowNum; i++) {
        rowId = input.readInt();
        nnz = input.readInt();
        totalNNZ = (int) (nnz * (model.col) / (partMeta.getEndCol() - partMeta.getStartCol()));
        row = model.getRow(rowId, partMeta.getPartId(), totalNNZ);
        for (int j = 0; j < nnz; j++) {
            row.put(input.readLong(), input.readDouble());
        }
    }
}

Example 33 with Long2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap in project angel by Tencent.

the class ModelLoader method main.

public static void main(String[] args) throws IOException {
    final Configuration conf = new Configuration();
    // load hadoop configuration
    String hadoopHomePath = System.getenv("HADOOP_HOME");
    if (hadoopHomePath == null) {
        LOG.warn("HADOOP_HOME is empty.");
    } else {
        conf.addResource(new Path(hadoopHomePath + "/etc/hadoop/yarn-site.xml"));
        conf.addResource(new Path(hadoopHomePath + "/etc/hadoop/hdfs-site.xml"));
    }
    String baseDir = "out";
    String denseDoubleModelPath = baseDir + "/dense_double";
    String sparseDoubleModelPath = baseDir + "/sparse_double";
    String denseFloatModelPath = baseDir + "/dense_float";
    String sparseFloatModelPath = baseDir + "/sparse_float";
    String denseIntModelPath = baseDir + "/dense_int";
    String sparseIntModelPath = baseDir + "/sparse_int";
    String sparseDoubleLongKeyModelPath = baseDir + "/sparse_double_longkey";
    double[][] denseDoubleModel = loadToDoubleArrays(denseDoubleModelPath, conf);
    int size = denseDoubleModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model dense_double row " + i + " sum is " + sum(denseDoubleModel[i]));
    }
    denseDoubleModel = null;
    Int2DoubleOpenHashMap[] sparseDoubleModel = loadToDoubleMaps(sparseDoubleModelPath, conf);
    size = sparseDoubleModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_double row " + i + " sum is " + sum(sparseDoubleModel[i]));
    }
    sparseDoubleModel = null;
    float[][] denseFloatModel = loadToFloatArrays(denseFloatModelPath, conf);
    size = denseFloatModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model dense_float row " + i + " sum is " + sum(denseFloatModel[i]));
    }
    denseFloatModel = null;
    Int2FloatOpenHashMap[] sparseFloatModel = loadToFloatMaps(sparseFloatModelPath, conf);
    size = sparseFloatModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_float row " + i + " sum is " + sum(sparseFloatModel[i]));
    }
    sparseFloatModel = null;
    int[][] denseIntModel = loadToIntArrays(denseIntModelPath, conf);
    size = denseIntModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model dense_int row " + i + " sum is " + sum(denseIntModel[i]));
    }
    denseIntModel = null;
    Int2IntOpenHashMap[] sparseIntModel = loadToIntMaps(sparseIntModelPath, conf);
    size = sparseIntModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_int row " + i + " sum is " + sum(sparseIntModel[i]));
    }
    sparseIntModel = null;
    Long2DoubleOpenHashMap[] sparseDoubleLongKeyModel = loadToDoubleLongKeyMaps(sparseDoubleLongKeyModelPath, conf);
    size = sparseDoubleLongKeyModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_double_longkey row " + i + " sum is " + sum(sparseDoubleLongKeyModel[i]));
    }
    sparseDoubleLongKeyModel = null;
}

Example 34 with Long2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap in project angel by Tencent.

the class GetCloseness method merge.

@Override
public GetResult merge(List<PartitionGetResult> partResults) {
    Long2DoubleOpenHashMap closenesses = new Long2DoubleOpenHashMap();
    for (PartitionGetResult r : partResults) {
        GetClosenessPartResult rr = (GetClosenessPartResult) r;
        closenesses.putAll(rr.getClosenesses());
    }
    return new GetClosenessResult(closenesses);
}

Example 35 with Long2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.longs.Long2DoubleOpenHashMap in project angel by Tencent.

the class GetCloseness method partitionGet.

@Override
public PartitionGetResult partitionGet(PartitionGetParam partParam) {
    GetHyperLogLogPartParam param = (GetHyperLogLogPartParam) partParam;
    ServerLongAnyRow row = GraphMatrixUtils.getPSLongKeyRow(psContext, param);
    ILongKeyPartOp keyPart = (ILongKeyPartOp) param.getNodes();
    long[] nodes = keyPart.getKeys();
    long n = param.getN();
    Long2DoubleOpenHashMap closenesses = new Long2DoubleOpenHashMap();
    for (int i = 0; i < nodes.length; i++) {
        HyperLogLogPlusElement hllElem = (HyperLogLogPlusElement) row.get(nodes[i]);
        if (hllElem.getCloseness() < n - 1) {
            closenesses.put(nodes[i], 0);
        } else {
            closenesses.put(nodes[i], (double) n / (double) hllElem.getCloseness());
        }
    }
    return new GetClosenessPartResult(closenesses);
}