Examples with ModifiableDoubleDBIDList de.lmu.ifi.dbs.elki.database.ids.ModifiableDoubleDBIDList used on opensource projects

Example 41 with ModifiableDoubleDBIDList

use of de.lmu.ifi.dbs.elki.database.ids.ModifiableDoubleDBIDList in project elki by elki-project.

the class RdKNNTree method bulkReverseKNNQueryForID.

public List<ModifiableDoubleDBIDList> bulkReverseKNNQueryForID(DBIDs ids, int k, SpatialPrimitiveDistanceFunction<? super O> distanceFunction, KNNQuery<O> knnQuery) {
    checkDistanceFunction(distanceFunction);
    if (k > settings.k_max) {
        throw new IllegalArgumentException("Parameter k is not supported, k > k_max: " + k + " > " + settings.k_max);
    }
    // get candidates
    Map<DBID, ModifiableDoubleDBIDList> candidateMap = new HashMap<>();
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        DBID id = DBIDUtil.deref(iter);
        candidateMap.put(id, DBIDUtil.newDistanceDBIDList());
    }
    doBulkReverseKNN(getRoot(), ids, candidateMap);
    if (k == settings.k_max) {
        List<ModifiableDoubleDBIDList> resultList = new ArrayList<>();
        for (ModifiableDoubleDBIDList candidates : candidateMap.values()) {
            candidates.sort();
            resultList.add(candidates);
        }
        return resultList;
    }
    // refinement of candidates, if k < k_max
    // perform a knn query for the candidates
    ArrayModifiableDBIDs candidateIDs = DBIDUtil.newArray();
    for (ModifiableDoubleDBIDList candidates : candidateMap.values()) {
        candidateIDs.addDBIDs(candidates);
    }
    candidateIDs.sort();
    List<? extends KNNList> knnLists = knnQuery.getKNNForBulkDBIDs(candidateIDs, k);
    // and add candidate c to the result if o is a knn of c
    List<ModifiableDoubleDBIDList> resultList = new ArrayList<>();
    for (DBID id : candidateMap.keySet()) {
        ModifiableDoubleDBIDList candidates = candidateMap.get(id);
        ModifiableDoubleDBIDList result = DBIDUtil.newDistanceDBIDList();
        for (DoubleDBIDListIter candidate = candidates.iter(); candidate.valid(); candidate.advance()) {
            int pos = candidateIDs.binarySearch(candidate);
            assert (pos >= 0);
            for (DoubleDBIDListIter qr = knnLists.get(pos).iter(); qr.valid(); qr.advance()) {
                if (DBIDUtil.equal(id, qr)) {
                    result.add(qr.doubleValue(), candidate);
                    break;
                }
            }
        }
        resultList.add(result);
    }
    return resultList;
}

Example 42 with ModifiableDoubleDBIDList

use of de.lmu.ifi.dbs.elki.database.ids.ModifiableDoubleDBIDList in project elki by elki-project.

the class SimplifiedCoverTree method bulkConstruct.

/**
 * Bulk-load the cover tree.
 *
 * This bulk-load is slightly simpler than the one used in the original
 * cover-tree source: We do not look back into the "far" set of candidates.
 *
 * @param cur Current routing object
 * @param maxScale Maximum scale
 * @param elems Candidates
 * @return Root node of subtree
 */
protected Node bulkConstruct(DBIDRef cur, int maxScale, ModifiableDoubleDBIDList elems) {
    assert (!elems.contains(cur));
    final double max = maxDistance(elems);
    final int scale = Math.min(distToScale(max) - 1, maxScale);
    final int nextScale = scale - 1;
    // elements remaining:
    if (max <= 0 || scale <= scaleBottom || elems.size() < truncate) {
        return new Node(cur, max, elems);
    }
    // Find neighbors in the cover of the current object:
    ModifiableDoubleDBIDList candidates = DBIDUtil.newDistanceDBIDList();
    excludeNotCovered(elems, scaleToDist(scale), candidates);
    // If no elements were not in the cover, build a compact tree:
    if (candidates.size() == 0) {
        LOG.warning("Scale not chosen appropriately? " + max + " " + scaleToDist(scale));
        return bulkConstruct(cur, nextScale, elems);
    }
    // We will have at least one other child, so build the parent:
    Node node = new Node(cur, max);
    // Routing element now is a singleton:
    final boolean curSingleton = elems.size() == 0;
    if (!curSingleton) {
        // Add node for the routing object:
        node.children.add(bulkConstruct(cur, nextScale, elems));
    }
    final double fmax = scaleToDist(nextScale);
    // Build additional cover nodes:
    for (DoubleDBIDListIter it = candidates.iter(); it.valid(); ) {
        assert (it.getOffset() == 0);
        DBID t = DBIDUtil.deref(it);
        // Recycle.
        elems.clear();
        collectByCover(it, candidates, fmax, elems);
        assert (DBIDUtil.equal(t, it)) : "First element in candidates must not change!";
        if (elems.size() == 0) {
            // Singleton
            node.singletons.add(it);
        } else {
            // Build a full child node:
            node.children.add(bulkConstruct(it, nextScale, elems));
        }
        candidates.removeSwap(0);
    }
    assert (candidates.size() == 0);
    // Routing object is not yet handled:
    if (curSingleton) {
        if (node.isLeaf()) {
            // First in leaf is enough.
            node.children = null;
        } else {
            // Add as regular singleton.
            node.singletons.add(cur);
        }
    }
    // TODO: improve recycling of lists?
    return node;
}

Example 43 with ModifiableDoubleDBIDList

use of de.lmu.ifi.dbs.elki.database.ids.ModifiableDoubleDBIDList in project elki by elki-project.

the class SimplifiedCoverTree method bulkLoad.

/**
 * Bulk-load the index.
 *
 * @param ids IDs to load
 */
public void bulkLoad(DBIDs ids) {
    if (ids.size() == 0) {
        return;
    }
    assert (root == null) : "Tree already initialized.";
    DBIDIter it = ids.iter();
    DBID first = DBIDUtil.deref(it);
    // Compute distances to all neighbors:
    ModifiableDoubleDBIDList candidates = DBIDUtil.newDistanceDBIDList(ids.size() - 1);
    for (it.advance(); it.valid(); it.advance()) {
        candidates.add(distance(first, it), it);
    }
    root = bulkConstruct(first, Integer.MAX_VALUE, candidates);
}

Example 44 with ModifiableDoubleDBIDList

use of de.lmu.ifi.dbs.elki.database.ids.ModifiableDoubleDBIDList in project elki by elki-project.

the class InMemoryInvertedIndex method initialize.

@Override
public void initialize() {
    if (index != null) {
        LOG.warning("Index was already initialized!");
    }
    index = new ArrayList<>();
    length = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_DB);
    for (DBIDIter iter = relation.iterDBIDs(); iter.valid(); iter.advance()) {
        V obj = relation.get(iter);
        if (obj instanceof SparseNumberVector) {
            indexSparse(iter, (SparseNumberVector) obj);
        } else {
            indexDense(iter, obj);
        }
    }
    // Sort indexes
    long count = 0L;
    for (ModifiableDoubleDBIDList column : index) {
        column.sort();
        count += column.size();
    }
    double sparsity = count / (index.size() * (double) relation.size());
    if (sparsity > .2) {
        LOG.warning("Inverted list indexes only perform well for very sparse data. Your data set has a sparsity of " + sparsity);
    }
}

Example 45 with ModifiableDoubleDBIDList

use of de.lmu.ifi.dbs.elki.database.ids.ModifiableDoubleDBIDList in project elki by elki-project.

the class CacheDoubleDistanceRangeQueries method run.

@Override
public void run() {
    database.initialize();
    Relation<O> relation = database.getRelation(distance.getInputTypeRestriction());
    DistanceQuery<O> distanceQuery = database.getDistanceQuery(relation, distance);
    RangeQuery<O> rangeQ = database.getRangeQuery(distanceQuery, radius, DatabaseQuery.HINT_HEAVY_USE);
    LOG.verbose("Performing range queries with radius " + radius);
    // open file.
    try (RandomAccessFile file = new RandomAccessFile(out, "rw");
        FileChannel channel = file.getChannel();
        // and acquire a file write lock
        FileLock lock = channel.lock()) {
        // write magic header
        file.writeInt(RANGE_CACHE_MAGIC);
        // write the query radius.
        file.writeDouble(radius);
        // Initial size, enough for 100.
        int bufsize = 100 * 12 * 2 + 10;
        ByteBuffer buffer = ByteBuffer.allocateDirect(bufsize);
        FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Computing range queries", relation.size(), LOG) : null;
        ModifiableDoubleDBIDList nn = DBIDUtil.newDistanceDBIDList();
        DoubleDBIDListIter ni = nn.iter();
        for (DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) {
            nn.clear();
            rangeQ.getRangeForDBID(it, radius, nn);
            nn.sort();
            final int nnsize = nn.size();
            // Grow the buffer when needed:
            if (nnsize * 12 + 10 > bufsize) {
                while (nnsize * 12 + 10 > bufsize) {
                    bufsize <<= 1;
                }
                LOG.verbose("Resizing buffer to " + bufsize + " to store " + nnsize + " results:");
                buffer = ByteBuffer.allocateDirect(bufsize);
            }
            buffer.clear();
            ByteArrayUtil.writeUnsignedVarint(buffer, it.internalGetIndex());
            ByteArrayUtil.writeUnsignedVarint(buffer, nnsize);
            int c = 0;
            for (ni.seek(0); ni.valid(); ni.advance(), c++) {
                ByteArrayUtil.writeUnsignedVarint(buffer, ni.internalGetIndex());
                buffer.putDouble(ni.doubleValue());
            }
            if (c != nn.size()) {
                throw new AbortException("Sizes did not agree. Cache is invalid.");
            }
            buffer.flip();
            channel.write(buffer);
            LOG.incrementProcessed(prog);
        }
        LOG.ensureCompleted(prog);
        lock.release();
    } catch (IOException e) {
        LOG.exception(e);
    }
// FIXME: close!
}