Examples with Int2LongOpenHashMap it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap used on opensource projects

Example 1 with Int2LongOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap in project angel by Tencent.

the class MatrixFormatImpl method initMatrix.

public Matrix initMatrix(MatrixFilesMeta matrixFilesMeta) {
    Map<Integer, MatrixPartitionMeta> partMetas = matrixFilesMeta.getPartMetas();
    Int2LongOpenHashMap rowIdToElemNumMap = new Int2LongOpenHashMap();
    for (MatrixPartitionMeta partMeta : partMetas.values()) {
        Map<Integer, RowPartitionMeta> rowMetas = partMeta.getRowMetas();
        for (Map.Entry<Integer, RowPartitionMeta> rowMetaEntry : rowMetas.entrySet()) {
            rowIdToElemNumMap.addTo(rowMetaEntry.getKey(), rowMetaEntry.getValue().getElementNum());
        }
    }
    RowType rowType = RowType.valueOf(matrixFilesMeta.getRowType());
    RowBasedMatrix matrix = rbMatrix(rowType, matrixFilesMeta.getRow(), matrixFilesMeta.getCol());
    ObjectIterator<Int2LongMap.Entry> iter = rowIdToElemNumMap.int2LongEntrySet().fastIterator();
    Int2LongMap.Entry entry;
    while (iter.hasNext()) {
        entry = iter.next();
        matrix.setRow(entry.getIntKey(), initRow(rowType, matrixFilesMeta.getCol(), entry.getLongValue()));
    }
    return matrix;
}

Example 2 with Int2LongOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap in project angel by Tencent.

the class BalanceInputFormat method createSplitsGreedy.

void createSplitsGreedy(Map<String, Set<OneBlockInfo>> nodeToBlocks, Map<OneBlockInfo, String[]> blockToNodes, Map<String, List<OneBlockInfo>> rackToBlocks, long totLength, int num, long minSizeNode, long minSizeRack, List<InputSplit> splits) {
    List<OneBlockInfo> blocks = new ArrayList<>(blockToNodes.keySet());
    Collections.sort(blocks, new Comparator<OneBlockInfo>() {

        @Override
        public int compare(OneBlockInfo o1, OneBlockInfo o2) {
            return -(int) (o1.length - o2.length);
        }
    });
    Int2LongOpenHashMap loads = new Int2LongOpenHashMap();
    Map<Integer, ArrayList<OneBlockInfo>> parts = new HashMap<>();
    Map<Integer, Set<String>> locations = new HashMap<>();
    for (int i = 0; i < num; i++) {
        parts.put(i, new ArrayList<>());
        locations.put(i, new HashSet<>());
    }
    for (OneBlockInfo blockInfo : blocks) {
        long min = Long.MAX_VALUE;
        int selectPart = -1;
        for (int s = 0; s < num; s++) {
            if (loads.get(s) < min) {
                min = loads.get(s);
                selectPart = s;
            }
        }
        loads.addTo(selectPart, blockInfo.length);
        parts.get(selectPart).add(blockInfo);
        for (String host : blockInfo.hosts) locations.get(selectPart).add(host);
    }
    int resultSplitNum = 0;
    for (Map.Entry<Integer, ArrayList<OneBlockInfo>> entry : parts.entrySet()) {
        if (entry.getValue().size() > 0) {
            resultSplitNum++;
            addCreatedSplit(splits, locations.get(entry.getKey()), entry.getValue());
        }
    }
    LOG.info("expect split number=" + num + ",actual split number=" + resultSplitNum);
}

Example 3 with Int2LongOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2LongOpenHashMap in project pinot by linkedin.

the class DefaultGroupKeyGenerator method purgeLongMapKeys.

/**
   * Helper method to purge group keys that got trimmed from the group by result.
   * Lazily builds a reverse map from id to array group key on the first call.
   *
   * @param groupKeys Group keys to purge
   */
private void purgeLongMapKeys(int[] groupKeys) {
    if (groupKeys == null || groupKeys.length == 0) {
        // Nothing to purge
        return;
    }
    // Lazily build the idToGroupKey reverse map only once, and then keep the two maps in-sync after that.
    if (_trimMode == TrimMode.OFF) {
        _idToGroupKey = new Int2LongOpenHashMap(_groupKeyToId.size());
        for (Long2IntMap.Entry entry : _groupKeyToId.long2IntEntrySet()) {
            _idToGroupKey.put(entry.getIntValue(), entry.getLongKey());
        }
        _trimMode = TrimMode.ON;
    }
    // Purge the specified keys
    for (int groupKey : groupKeys) {
        _groupKeyToId.remove(_idToGroupKey.get(groupKey));
        _idToGroupKey.remove(groupKey);
    }
}