Examples with IntOpenHashSet it.unimi.dsi.fastutil.ints.IntOpenHashSet used on opensource projects

Example 16 with IntOpenHashSet

use of it.unimi.dsi.fastutil.ints.IntOpenHashSet in project gradle by gradle.

the class ClassSetAnalysis method findChangedConstants.

private Map<String, IntSet> findChangedConstants(ClassSetAnalysis other, DependentsSet affectedClasses) {
    if (affectedClasses.isDependencyToAll()) {
        return Collections.emptyMap();
    }
    Set<String> dependentClasses = affectedClasses.getAllDependentClasses();
    Map<String, IntSet> result = new HashMap<>(dependentClasses.size());
    for (String affectedClass : dependentClasses) {
        IntSet difference = new IntOpenHashSet(other.getConstants(affectedClass));
        difference.removeAll(getConstants(affectedClass));
        result.put(affectedClass, difference);
    }
    return result;
}

Example 17 with IntOpenHashSet

use of it.unimi.dsi.fastutil.ints.IntOpenHashSet in project angel by Tencent.

the class AMMatrixMetaManager method syncMatrixInfos.

/**
 * compare the matrix meta on the master and the matrix meta on ps to find the matrix this
 * parameter server needs to create and delete
 *
 * @param matrixReports parameter server matrix report, include the matrix ids this parameter
 * server hold.
 * @param needCreateMatrixes use to return the matrix partitions this parameter server need to
 * build
 * @param needReleaseMatrixes use to return the matrix ids this parameter server need to remove
 * @param needRecoverParts need recover partitions
 * @param psId parameter server id
 */
public void syncMatrixInfos(List<MatrixReport> matrixReports, List<MatrixMeta> needCreateMatrixes, List<Integer> needReleaseMatrixes, List<RecoverPartKey> needRecoverParts, ParameterServerId psId) {
    // get matrix ids in the parameter server report
    IntOpenHashSet matrixInPS = new IntOpenHashSet();
    int size = matrixReports.size();
    for (int i = 0; i < size; i++) {
        matrixInPS.add(matrixReports.get(i).matrixId);
    }
    handleMatrixReports(psId, matrixReports);
    Set<RecoverPartKey> parts = getAndRemoveNeedRecoverParts(psId);
    if (parts != null) {
        needRecoverParts.addAll(parts);
    }
    // get the matrices parameter server need to create and delete
    getPSNeedUpdateMatrix(matrixInPS, needCreateMatrixes, needReleaseMatrixes, psId);
    psMatricesUpdate(psId, matrixReports);
}

Example 18 with IntOpenHashSet

use of it.unimi.dsi.fastutil.ints.IntOpenHashSet in project angel by Tencent.

the class PSAgentMatrixMetaManager method getPartitionToRowIndexMap.

/**
 * Get the partitions the rows in.
 *
 * @param rowIndex    matrix id and row indexes
 * @param batchNumber the split batch size.
 * @return Map<PartitionKey, List<Integer>> partitions to the rows contained in the partition map
 */
public Map<PartitionKey, List<RowIndex>> getPartitionToRowIndexMap(RowIndex rowIndex, int batchNumber) {
    Map<PartitionKey, List<RowIndex>> partToRowIndexMap = new TreeMap<PartitionKey, List<RowIndex>>();
    if (rowIndex.getRowIds() == null) {
        return partToRowIndexMap;
    }
    IntOpenHashSet rowIdSet = rowIndex.getRowIds();
    IntOpenHashSet filtedRowIdSet = rowIndex.getFiltedIdSet();
    int[] rowIds = new int[rowIdSet.size() - filtedRowIdSet.size()];
    int count = 0;
    for (int rowId : rowIdSet) {
        if (!filtedRowIdSet.contains(rowId)) {
            rowIds[count++] = rowId;
        }
    }
    Arrays.sort(rowIds);
    int partNum = 0;
    for (int i = 0; i < rowIds.length; i++) {
        List<PartitionKey> partKeys = getPartitions(rowIndex.getMatrixId(), rowIds[i]);
        partNum = partKeys.size();
        for (int j = 0; j < partNum; j++) {
            PartitionKey partitionKey = partKeys.get(j);
            List<RowIndex> indexList = partToRowIndexMap.get(partitionKey);
            if (indexList == null) {
                indexList = new ArrayList<RowIndex>();
                partToRowIndexMap.put(partitionKey, indexList);
                indexList.add(new RowIndex(rowIndex.getMatrixId(), rowIndex));
            }
            RowIndex index = indexList.get(indexList.size() - 1);
            if (index.getRowsNumber() >= batchNumber) {
                index = new RowIndex(rowIndex.getMatrixId(), rowIndex);
                indexList.add(index);
            }
            index.addRowId(rowIds[i]);
        }
    }
    return partToRowIndexMap;
}

Example 19 with IntOpenHashSet

use of it.unimi.dsi.fastutil.ints.IntOpenHashSet in project angel by Tencent.

the class TaskManager method combineUpdateIndex.

/**
 * Combine update index.
 */
@SuppressWarnings("rawtypes")
public void combineUpdateIndex() {
    IntOpenHashSet indexSet = null;
    MatrixMeta meta = null;
    for (Entry<TaskId, Task> entry : runningTask.entrySet()) {
        LabeledUpdateIndexBaseTask task = (LabeledUpdateIndexBaseTask) entry.getValue().getUserTask();
        IntOpenHashSet taskIndexSet = task.getIndexSet();
        if (taskIndexSet != null) {
            if (indexSet == null) {
                indexSet = taskIndexSet;
                meta = task.getMatrixMeta();
            } else {
                indexSet.addAll(taskIndexSet);
                task.setIndexSet(null);
            }
        }
    }
    if (indexSet != null && meta != null) {
        int size = indexSet.size();
        int[] indexArray = new int[size];
        int index = 0;
        IntIterator iter = indexSet.iterator();
        while (iter.hasNext()) {
            indexArray[index++] = iter.nextInt();
        }
        Arrays.sort(indexArray);
        List<PartitionKey> partKeyList = WorkerContext.get().getPSAgent().getMatrixMetaManager().getPartitions(meta.getId());
        Collections.sort(partKeyList);
        int partNum = partKeyList.size();
        int lastPos = 0;
        for (int i = 0; i < partNum; i++) {
            PartitionKey partKey = partKeyList.get(i);
            long endCol = partKey.getEndCol();
            for (int j = lastPos; j < size; j++) {
                if (indexArray[j] >= endCol) {
                    lastPos = j;
                    break;
                }
            }
        }
    // Bitmap bitmap = new Bitmap();
    // int max = indexArray[size - 1];
    // byte [] bitIndexArray = new byte[max / 8 + 1];
    // for(int i = 0; i < size; i++){
    // int bitIndex = indexArray[i] >> 3;
    // int bitOffset = indexArray[i] - (bitIndex << 3);
    // switch(bitOffset){
    // case 0:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x01);break;
    // case 1:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x02);break;
    // case 2:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x04);break;
    // case 3:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x08);break;
    // case 4:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x10);break;
    // case 5:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x20);break;
    // case 6:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x40);break;
    // case 7:bitIndexArray[bitIndex] = (byte) (bitIndexArray[bitIndex] & 0x80);break;
    // }
    // }
    }
}

Example 20 with IntOpenHashSet

use of it.unimi.dsi.fastutil.ints.IntOpenHashSet in project angel by Tencent.

the class LINEFirstOrderModel method dot.

@Override
public float[] dot(ByteBuf edges) {
    Random negativeSeed = new Random(seed);
    IntOpenHashSet numInputs = new IntOpenHashSet();
    int batchSize = edges.readInt();
    float[] partialDots = new float[batchSize * (1 + negative)];
    int dotInc = 0;
    for (int position = 0; position < batchSize; position++) {
        int src = edges.readInt();
        int dst = edges.readInt();
        // Skip-Gram model
        float[] inputs = layers[src / numNodeOneRow];
        int l1 = (src % numNodeOneRow) * dim;
        numInputs.add(src);
        // Negative sampling
        int target;
        for (int a = 0; a < negative + 1; a++) {
            if (a == 0)
                target = dst;
            else
                do {
                    target = negativeSeed.nextInt(maxIndex);
                } while (target == src);
            numInputs.add(target);
            float[] outputs = layers[target / numNodeOneRow];
            int l2 = (target % numNodeOneRow) * dim;
            float f = 0.0f;
            for (int b = 0; b < dim; b++) f += inputs[l1 + b] * outputs[l2 + b];
            partialDots[dotInc++] = f;
        }
    }
    this.numInputsToUpdate = numInputs.size();
    return partialDots;
}