Examples with Int2DoubleOpenHashMap it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap used on opensource projects

Example 1 with Int2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap in project pinot by linkedin.

the class DoubleGroupByResultHolder method switchToMapMode.

/**
   * Helper method to switch the storage from array mode to map mode.
   *
   * @param initialPriorityQueueSize Initial size of priority queue
   */
private void switchToMapMode(int initialPriorityQueueSize) {
    _storageMode = StorageMode.MAP_STORAGE;
    _resultMap = new Int2DoubleOpenHashMap(_resultHolderCapacity);
    _priorityQueue = new IntDoubleIndexedPriorityQueue(initialPriorityQueueSize, _minHeap);
    for (int id = 0; id < _resultHolderCapacity; id++) {
        _resultMap.put(id, _resultArray[id]);
        _priorityQueue.put(id, _resultArray[id]);
    }
}

Example 2 with Int2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap in project pinot by linkedin.

the class IntDoubleIndexedPriorityQueueTest method test.

/**
   * Helper method builds the priority queue, randomly updates elements and
   * then asserts the following:
   * <ul>
   *   <li> Elements are popped from the priority queue in the expected order. </li>
   *   <li> Size of the priority queue is as expected (after elements are updated). </li>
   * </ul>
   * @param minHeap Min or max mode
   */
public void test(boolean minHeap) {
    Random random = new Random(0);
    IntDoubleIndexedPriorityQueue pq = new IntDoubleIndexedPriorityQueue(NUM_RECORDS, minHeap);
    Int2DoubleOpenHashMap map = new Int2DoubleOpenHashMap(NUM_RECORDS);
    // Initialize the priority queue.
    for (int i = 0; i < NUM_RECORDS; i++) {
        double value = random.nextDouble();
        pq.put(i, value);
        map.put(i, value);
    }
    // Update some records randomly
    for (int i = 0; i < NUM_RECORDS; i++) {
        int key = random.nextInt(NUM_RECORDS);
        double value = random.nextDouble();
        pq.put(key, value);
        map.put(key, value);
    }
    // Transfer the map into list so it can be sorted.
    List<Pairs.IntDoublePair> list = new ArrayList<>(NUM_RECORDS);
    for (Int2DoubleMap.Entry entry : map.int2DoubleEntrySet()) {
        list.add(new Pairs.IntDoublePair(entry.getKey(), entry.getValue()));
    }
    // Comparison for min heap is the same as that for ascending order.
    boolean descendingOrder = !minHeap;
    Collections.sort(list, new Pairs.IntDoubleComparator(descendingOrder));
    // Ensure that elements are popped from priority queue in the expected order.
    int i = 0;
    while (!pq.isEmpty()) {
        Pairs.IntDoublePair actual = pq.poll();
        Pairs.IntDoublePair expected = list.get(i++);
        Assert.assertEquals(actual.getIntValue(), expected.getIntValue());
        Assert.assertEquals(actual.getDoubleValue(), expected.getDoubleValue());
    }
    // Assert that priority queue had expected number of elements.
    Assert.assertEquals(i, list.size());
}

Example 3 with Int2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap in project ignite by apache.

the class ColumnDecisionTreeTrainerBenchmark method loadVectorsIntoSparseDistributedMatrixCache.

/**
 * Load vectors into sparse distributed matrix.
 *
 * @param cacheName Name of cache where matrix is stored.
 * @param uuid UUID of matrix.
 * @param iter Iterator over vectors.
 * @param vectorSize size of vectors.
 */
private void loadVectorsIntoSparseDistributedMatrixCache(String cacheName, UUID uuid, Iterator<? extends org.apache.ignite.ml.math.Vector> iter, int vectorSize) {
    try (IgniteDataStreamer<SparseMatrixKey, Map<Integer, Double>> streamer = Ignition.localIgnite().dataStreamer(cacheName)) {
        int sampleIdx = 0;
        streamer.allowOverwrite(true);
        streamer.receiver(StreamTransformer.from((e, arg) -> {
            Map<Integer, Double> val = e.getValue();
            if (val == null)
                val = new Int2DoubleOpenHashMap();
            val.putAll((Map<Integer, Double>) arg[0]);
            e.setValue(val);
            return null;
        }));
        // Feature index -> (sample index -> value)
        Map<Integer, Map<Integer, Double>> batch = new HashMap<>();
        IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
        int batchSize = 1000;
        while (iter.hasNext()) {
            org.apache.ignite.ml.math.Vector next = iter.next();
            for (int i = 0; i < vectorSize; i++) batch.get(i).put(sampleIdx, next.getX(i));
            X.println("Sample index: " + sampleIdx);
            if (sampleIdx % batchSize == 0) {
                batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
                IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
            }
            sampleIdx++;
        }
        if (sampleIdx % batchSize != 0) {
            batch.keySet().forEach(fi -> streamer.addData(new SparseMatrixKey(fi, uuid, fi), batch.get(fi)));
            IntStream.range(0, vectorSize).forEach(i -> batch.put(i, new HashMap<>()));
        }
    }
}

Example 4 with Int2DoubleOpenHashMap

use of it.unimi.dsi.fastutil.ints.Int2DoubleOpenHashMap in project ignite by apache.

the class SparseDistributedMatrixStorage method matrixSet.

/**
 * Distributed matrix set.
 *
 * @param a Row or column index.
 * @param b Row or column index.
 * @param v New value to set.
 */
private void matrixSet(int a, int b, double v) {
    // Remote set on the primary node (where given row or column is stored locally).
    ignite().compute(getClusterGroupForGivenKey(CACHE_NAME, a)).run(() -> {
        IgniteCache<RowColMatrixKey, Map<Integer, Double>> cache = Ignition.localIgnite().getOrCreateCache(CACHE_NAME);
        // Local get.
        Map<Integer, Double> map = cache.localPeek(getCacheKey(a), CachePeekMode.PRIMARY);
        if (map == null) {
            // Remote entry get.
            map = cache.get(getCacheKey(a));
            if (map == null)
                map = acsMode == SEQUENTIAL_ACCESS_MODE ? new Int2DoubleRBTreeMap() : new Int2DoubleOpenHashMap();
        }
        if (v != 0.0)
            map.put(b, v);
        else if (map.containsKey(b))
            map.remove(b);
        // Local put.
        cache.put(getCacheKey(a), map);
    });
}