Examples with Int2FloatOpenHashMap it.unimi.dsi.fastutil.ints.Int2FloatOpenHashMap used on opensource projects

Example 11 with Int2FloatOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2FloatOpenHashMap in project angel by Tencent.

the class ModelLoader method main.

public static void main(String[] args) throws IOException {
    final Configuration conf = new Configuration();
    // load hadoop configuration
    String hadoopHomePath = System.getenv("HADOOP_HOME");
    if (hadoopHomePath == null) {
        LOG.warn("HADOOP_HOME is empty.");
    } else {
        conf.addResource(new Path(hadoopHomePath + "/etc/hadoop/yarn-site.xml"));
        conf.addResource(new Path(hadoopHomePath + "/etc/hadoop/hdfs-site.xml"));
    }
    String baseDir = "out";
    String denseDoubleModelPath = baseDir + "/dense_double";
    String sparseDoubleModelPath = baseDir + "/sparse_double";
    String denseFloatModelPath = baseDir + "/dense_float";
    String sparseFloatModelPath = baseDir + "/sparse_float";
    String denseIntModelPath = baseDir + "/dense_int";
    String sparseIntModelPath = baseDir + "/sparse_int";
    String sparseDoubleLongKeyModelPath = baseDir + "/sparse_double_longkey";
    double[][] denseDoubleModel = loadToDoubleArrays(denseDoubleModelPath, conf);
    int size = denseDoubleModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model dense_double row " + i + " sum is " + sum(denseDoubleModel[i]));
    }
    denseDoubleModel = null;
    Int2DoubleOpenHashMap[] sparseDoubleModel = loadToDoubleMaps(sparseDoubleModelPath, conf);
    size = sparseDoubleModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_double row " + i + " sum is " + sum(sparseDoubleModel[i]));
    }
    sparseDoubleModel = null;
    float[][] denseFloatModel = loadToFloatArrays(denseFloatModelPath, conf);
    size = denseFloatModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model dense_float row " + i + " sum is " + sum(denseFloatModel[i]));
    }
    denseFloatModel = null;
    Int2FloatOpenHashMap[] sparseFloatModel = loadToFloatMaps(sparseFloatModelPath, conf);
    size = sparseFloatModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_float row " + i + " sum is " + sum(sparseFloatModel[i]));
    }
    sparseFloatModel = null;
    int[][] denseIntModel = loadToIntArrays(denseIntModelPath, conf);
    size = denseIntModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model dense_int row " + i + " sum is " + sum(denseIntModel[i]));
    }
    denseIntModel = null;
    Int2IntOpenHashMap[] sparseIntModel = loadToIntMaps(sparseIntModelPath, conf);
    size = sparseIntModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_int row " + i + " sum is " + sum(sparseIntModel[i]));
    }
    sparseIntModel = null;
    Long2DoubleOpenHashMap[] sparseDoubleLongKeyModel = loadToDoubleLongKeyMaps(sparseDoubleLongKeyModelPath, conf);
    size = sparseDoubleLongKeyModel.length;
    for (int i = 0; i < size; i++) {
        LOG.info("model sparse_double_longkey row " + i + " sum is " + sum(sparseDoubleLongKeyModel[i]));
    }
    sparseDoubleLongKeyModel = null;
}

Example 12 with Int2FloatOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2FloatOpenHashMap in project angel by Tencent.

the class ModelLoader method loadSparseFloatPartition.

private static void loadSparseFloatPartition(SparseFloatModel model, FSDataInputStream input, MatrixPartitionMeta partMeta) throws IOException {
    int rowNum = input.readInt();
    int rowId = 0;
    int nnz = 0;
    int totalNNZ = 0;
    Int2FloatOpenHashMap row = null;
    for (int i = 0; i < rowNum; i++) {
        rowId = input.readInt();
        nnz = input.readInt();
        totalNNZ = (int) (nnz * (model.col) / (partMeta.getEndCol() - partMeta.getStartCol()));
        row = model.getRow(rowId, partMeta.getPartId(), totalNNZ);
        for (int j = 0; j < nnz; j++) {
            row.put(input.readInt(), input.readFloat());
        }
    }
}