Examples with KNNList de.lmu.ifi.dbs.elki.database.ids.KNNList used on opensource projects

Example 71 with KNNList

use of de.lmu.ifi.dbs.elki.database.ids.KNNList in project elki by elki-project.

the class KNNDD method run.

/**
 * Runs the algorithm in the timed evaluation part.
 *
 * @param relation Data relation
 */
public OutlierResult run(Relation<O> relation) {
    final DistanceQuery<O> distanceQuery = relation.getDistanceQuery(getDistanceFunction());
    final KNNQuery<O> knnQuery = relation.getKNNQuery(distanceQuery, k);
    FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("kNN distance for objects", relation.size(), LOG) : null;
    WritableDoubleDataStore knnDist = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
    WritableDBIDDataStore neighbor = DataStoreUtil.makeDBIDStorage(relation.getDBIDs(), DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
    DBIDVar var = DBIDUtil.newVar();
    // Find nearest neighbors, and store the distances.
    for (DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) {
        final KNNList knn = knnQuery.getKNNForDBID(it, k);
        knnDist.putDouble(it, knn.getKNNDistance());
        neighbor.put(it, knn.assignVar(knn.size() - 1, var));
        LOG.incrementProcessed(prog);
    }
    LOG.ensureCompleted(prog);
    prog = LOG.isVerbose() ? new FiniteProgress("kNN distance descriptor", relation.size(), LOG) : null;
    WritableDoubleDataStore scores = DataStoreUtil.makeDoubleStorage(relation.getDBIDs(), DataStoreFactory.HINT_DB);
    DoubleMinMax minmax = new DoubleMinMax();
    for (DBIDIter it = relation.iterDBIDs(); it.valid(); it.advance()) {
        // Distance
        double d = knnDist.doubleValue(it);
        // Distance of neighbor
        double nd = knnDist.doubleValue(neighbor.assignVar(it, var));
        double knndd = nd > 0 ? d / nd : d > 0 ? Double.POSITIVE_INFINITY : 1.;
        scores.put(it, knndd);
        minmax.put(knndd);
        LOG.incrementProcessed(prog);
    }
    LOG.ensureCompleted(prog);
    DoubleRelation scoreres = new MaterializedDoubleRelation("kNN Data Descriptor", "knndd-outlier", scores, relation.getDBIDs());
    OutlierScoreMeta meta = new BasicOutlierScoreMeta(minmax.getMin(), minmax.getMax(), 0., Double.POSITIVE_INFINITY, 1.);
    return new OutlierResult(meta, scoreres);
}

Example 72 with KNNList

use of de.lmu.ifi.dbs.elki.database.ids.KNNList in project elki by elki-project.

the class MaterializeKNNAndRKNNPreprocessor method materializeKNNAndRKNNs.

/**
 * Materializes the kNNs and RkNNs of the specified object IDs.
 *
 * @param ids the IDs of the objects
 */
private void materializeKNNAndRKNNs(ArrayDBIDs ids, FiniteProgress progress) {
    // add an empty list to each rknn
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        if (materialized_RkNN.get(iter) == null) {
            materialized_RkNN.put(iter, new TreeSet<DoubleDBIDPair>());
        }
    }
    // knn query
    List<? extends KNNList> kNNList = knnQuery.getKNNForBulkDBIDs(ids, k);
    int i = 0;
    for (DBIDIter id = ids.iter(); id.valid(); id.advance(), i++) {
        KNNList kNNs = kNNList.get(i);
        storage.put(id, kNNs);
        for (DoubleDBIDListIter iter = kNNs.iter(); iter.valid(); iter.advance()) {
            TreeSet<DoubleDBIDPair> rknns = materialized_RkNN.get(iter);
            rknns.add(makePair(iter, id));
        }
        getLogger().incrementProcessed(progress);
    }
    getLogger().ensureCompleted(progress);
}

Example 73 with KNNList

use of de.lmu.ifi.dbs.elki.database.ids.KNNList in project elki by elki-project.

the class MaterializeKNNAndRKNNPreprocessor method updateKNNsAndRkNNs.

/**
 * Updates the kNNs and RkNNs after insertion of the specified ids.
 *
 * @param ids the ids of newly inserted objects causing a change of
 *        materialized kNNs and RkNNs
 * @return the RkNNs of the specified ids, i.e. the kNNs which have been
 *         updated
 */
private ArrayDBIDs updateKNNsAndRkNNs(DBIDs ids) {
    ArrayModifiableDBIDs rkNN_ids = DBIDUtil.newArray();
    DBIDs oldids = DBIDUtil.difference(relation.getDBIDs(), ids);
    for (DBIDIter id = oldids.iter(); id.valid(); id.advance()) {
        KNNList oldkNNs = storage.get(id);
        double knnDist = oldkNNs.getKNNDistance();
        // look for new kNNs
        KNNHeap heap = null;
        for (DBIDIter newid = ids.iter(); newid.valid(); newid.advance()) {
            double dist = distanceQuery.distance(id, newid);
            if (dist <= knnDist) {
                // New id changes the kNNs of oldid.
                if (heap == null) {
                    heap = DBIDUtil.newHeap(oldkNNs);
                }
                heap.insert(dist, newid);
            }
        }
        // kNNs for oldid have changed:
        if (heap != null) {
            KNNList newkNNs = heap.toKNNList();
            storage.put(id, newkNNs);
            // get the difference
            int i = 0;
            int j = 0;
            ModifiableDoubleDBIDList added = DBIDUtil.newDistanceDBIDList();
            ModifiableDoubleDBIDList removed = DBIDUtil.newDistanceDBIDList();
            // TODO: use iterators.
            while (i < oldkNNs.size() && j < newkNNs.size()) {
                DoubleDBIDPair drp1 = oldkNNs.get(i);
                DoubleDBIDPair drp2 = newkNNs.get(j);
                // NOTE: we assume that on ties they are ordered the same way!
                if (!DBIDUtil.equal(drp1, drp2)) {
                    added.add(drp2);
                    j++;
                } else {
                    i++;
                    j++;
                }
            }
            if (i != j) {
                for (; i < oldkNNs.size(); i++) {
                    removed.add(oldkNNs.get(i));
                }
                for (; j < newkNNs.size(); i++) {
                    added.add(newkNNs.get(i));
                }
            }
            // add new RkNN
            for (DoubleDBIDListIter newnn = added.iter(); newnn.valid(); newnn.advance()) {
                TreeSet<DoubleDBIDPair> rknns = materialized_RkNN.get(newnn);
                rknns.add(makePair(newnn, id));
            }
            // remove old RkNN
            for (DoubleDBIDListIter oldnn = removed.iter(); oldnn.valid(); oldnn.advance()) {
                TreeSet<DoubleDBIDPair> rknns = materialized_RkNN.get(oldnn);
                rknns.remove(makePair(oldnn, id));
            }
            rkNN_ids.add(id);
        }
    }
    return rkNN_ids;
}

Example 74 with KNNList

use of de.lmu.ifi.dbs.elki.database.ids.KNNList in project elki by elki-project.

the class NNDescentTest method testKNNQueries.

private void testKNNQueries(Relation<DoubleVector> rep, KNNQuery<DoubleVector> lin_knn_query, KNNQuery<DoubleVector> preproc_knn_query, int k) {
    ArrayDBIDs sample = DBIDUtil.ensureArray(rep.getDBIDs());
    List<? extends KNNList> lin_knn_ids = lin_knn_query.getKNNForBulkDBIDs(sample, k);
    List<? extends KNNList> preproc_knn_ids = preproc_knn_query.getKNNForBulkDBIDs(sample, k);
    for (int i = 0; i < rep.size(); i++) {
        KNNList lin_knn = lin_knn_ids.get(i);
        KNNList pre_knn = preproc_knn_ids.get(i);
        DoubleDBIDListIter lin = lin_knn.iter(), pre = pre_knn.iter();
        for (; lin.valid() && pre.valid(); lin.advance(), pre.advance(), i++) {
            if (DBIDUtil.equal(lin, pre) || lin.doubleValue() == pre.doubleValue()) {
                continue;
            }
            fail(// 
            new StringBuilder(1000).append("Neighbor distances do not agree: ").append(lin_knn.toString()).append(" got: ").append(pre_knn.toString()).toString());
        }
        assertEquals("kNN sizes do not agree.", lin_knn.size(), pre_knn.size());
        for (int j = 0; j < lin_knn.size(); j++) {
            assertTrue("kNNs of linear scan and preprocessor do not match!", DBIDUtil.equal(lin_knn.get(j), pre_knn.get(j)));
            assertEquals("kNNs of linear scan and preprocessor do not match!", lin_knn.get(j).doubleValue(), pre_knn.get(j).doubleValue(), 0.);
        }
    }
}

Example 75 with KNNList

use of de.lmu.ifi.dbs.elki.database.ids.KNNList in project elki by elki-project.

the class MkMaxTree method preInsert.

/**
 * Adapts the knn distances before insertion of entry q.
 *
 * @param q the entry to be inserted
 * @param nodeEntry the entry representing the root of the current subtree
 * @param knns_q the knns of q
 */
private void preInsert(MkMaxEntry q, MkMaxEntry nodeEntry, KNNHeap knns_q) {
    if (LOG.isDebugging()) {
        LOG.debugFine("preInsert " + q + " - " + nodeEntry + "\n");
    }
    double knnDist_q = knns_q.getKNNDistance();
    MkMaxTreeNode<O> node = getNode(nodeEntry);
    double knnDist_node = 0.;
    // leaf node
    if (node.isLeaf()) {
        for (int i = 0; i < node.getNumEntries(); i++) {
            MkMaxEntry p = node.getEntry(i);
            double dist_pq = distance(p.getRoutingObjectID(), q.getRoutingObjectID());
            // ==> p becomes a knn-candidate
            if (dist_pq <= knnDist_q) {
                knns_q.insert(dist_pq, p.getRoutingObjectID());
                if (knns_q.size() >= getKmax()) {
                    knnDist_q = knns_q.getKNNDistance();
                    q.setKnnDistance(knnDist_q);
                }
            }
            // q becomes knn of p
            if (dist_pq <= p.getKnnDistance()) {
                KNNList knns_p = knnq.getKNNForDBID(p.getRoutingObjectID(), getKmax() - 1);
                if (knns_p.size() + 1 < getKmax()) {
                    p.setKnnDistance(Double.NaN);
                } else {
                    double knnDist_p = Math.max(dist_pq, knns_p.getKNNDistance());
                    p.setKnnDistance(knnDist_p);
                }
            }
            knnDist_node = Math.max(knnDist_node, p.getKnnDistance());
        }
    } else // directory node
    {
        List<DoubleIntPair> entries = getSortedEntries(node, q.getRoutingObjectID());
        for (DoubleIntPair distEntry : entries) {
            MkMaxEntry dirEntry = node.getEntry(distEntry.second);
            double entry_knnDist = dirEntry.getKnnDistance();
            if (distEntry.second < entry_knnDist || distEntry.second < knnDist_q) {
                preInsert(q, dirEntry, knns_q);
                knnDist_q = knns_q.getKNNDistance();
            }
            knnDist_node = Math.max(knnDist_node, dirEntry.getKnnDistance());
        }
    }
    if (LOG.isDebugging()) {
        LOG.debugFine(nodeEntry + "set knn dist " + knnDist_node);
    }
    nodeEntry.setKnnDistance(knnDist_node);
}