Examples with Logging de.lmu.ifi.dbs.elki.logging.Logging used on opensource projects

Example 1 with Logging

use of de.lmu.ifi.dbs.elki.logging.Logging in project elki by elki-project.

the class AbstractHDBSCAN method convertToPointerRepresentation.

/**
 * Convert spanning tree to a pointer representation.
 *
 * Note: the heap must use the correct encoding of indexes.
 *
 * @param ids IDs indexed
 * @param heap Heap
 * @param pi Parent array
 * @param lambda Distance array
 */
protected void convertToPointerRepresentation(ArrayDBIDs ids, DoubleLongHeap heap, WritableDBIDDataStore pi, WritableDoubleDataStore lambda) {
    final Logging LOG = getLogger();
    // Initialize parent array:
    for (DBIDArrayIter iter = ids.iter(); iter.valid(); iter.advance()) {
        // Initialize
        pi.put(iter, iter);
    }
    DBIDVar p = DBIDUtil.newVar(), q = DBIDUtil.newVar(), n = DBIDUtil.newVar();
    FiniteProgress pprog = LOG.isVerbose() ? new FiniteProgress("Converting MST to pointer representation", heap.size(), LOG) : null;
    while (!heap.isEmpty()) {
        final double dist = heap.peekKey();
        final long pair = heap.peekValue();
        final int i = (int) (pair >>> 31), j = (int) (pair & 0x7FFFFFFFL);
        ids.assignVar(i, p);
        // Follow p to its parent.
        while (!DBIDUtil.equal(p, pi.assignVar(p, n))) {
            p.set(n);
        }
        // Follow q to its parent.
        ids.assignVar(j, q);
        while (!DBIDUtil.equal(q, pi.assignVar(q, n))) {
            q.set(n);
        }
        // By definition of the pointer representation, the largest element in
        // each cluster is the cluster lead.
        // The extraction methods currently rely on this!
        int c = DBIDUtil.compare(p, q);
        if (c < 0) {
            // p joins q:
            pi.put(p, q);
            lambda.put(p, dist);
        } else {
            assert (c != 0) : "This should never happen!";
            // q joins p:
            pi.put(q, p);
            lambda.put(q, dist);
        }
        heap.poll();
        LOG.incrementProcessed(pprog);
    }
    LOG.ensureCompleted(pprog);
    // does not fulfill the property that the last element has the largest id.
    for (DBIDArrayIter iter = ids.iter(); iter.valid(); iter.advance()) {
        double d = lambda.doubleValue(iter);
        // Parent:
        pi.assignVar(iter, p);
        q.set(p);
        // Follow parent while tied.
        while (d >= lambda.doubleValue(q) && !DBIDUtil.equal(q, pi.assignVar(q, n))) {
            q.set(n);
        }
        if (!DBIDUtil.equal(p, q)) {
            if (LOG.isDebuggingFinest()) {
                LOG.finest("Correcting parent: " + p + " -> " + q);
            }
            pi.put(iter, q);
        }
    }
}

Example 2 with Logging

use of de.lmu.ifi.dbs.elki.logging.Logging in project elki by elki-project.

the class MaterializeKNNPreprocessor method objectsRemoved.

/**
 * Called after objects have been removed, updates the materialized
 * neighborhood.
 *
 * @param ids the ids of the removed objects
 */
protected void objectsRemoved(DBIDs ids) {
    final Logging log = getLogger();
    StepProgress stepprog = log.isVerbose() ? new StepProgress(3) : null;
    // delete the materialized (old) kNNs
    log.beginStep(stepprog, 1, "New deletions ocurred, remove their materialized kNNs.");
    for (DBIDIter iter = ids.iter(); iter.valid(); iter.advance()) {
        storage.delete(iter);
    }
    // update the affected kNNs
    log.beginStep(stepprog, 2, "New deletions ocurred, update the affected kNNs.");
    ArrayDBIDs rkNN_ids = updateKNNsAfterDeletion(ids);
    // inform listener
    log.beginStep(stepprog, 3, "New deletions ocurred, inform listeners.");
    fireKNNsRemoved(ids, rkNN_ids);
    log.ensureCompleted(stepprog);
}

Example 3 with Logging

use of de.lmu.ifi.dbs.elki.logging.Logging in project elki by elki-project.

the class AbstractMTree method adjustTree.

/**
 * Adjusts the tree after insertion of some nodes.
 *
 * @param subtree the subtree to be adjusted
 */
private void adjustTree(IndexTreePath<E> subtree) {
    final Logging log = getLogger();
    if (log.isDebugging()) {
        log.debugFine("Adjust tree " + subtree + "\n");
    }
    // get the root of the subtree
    int nodeIndex = subtree.getIndex();
    N node = getNode(subtree.getEntry());
    // overflow in node; split the node
    if (hasOverflow(node)) {
        // do the split
        Assignments<E> assignments = settings.splitStrategy.split(this, node);
        final N newNode = node.isLeaf() ? createNewLeafNode() : createNewDirectoryNode();
        List<E> entries1 = new ArrayList<>(assignments.getFirstAssignments().size());
        List<E> entries2 = new ArrayList<>(assignments.getSecondAssignments().size());
        // Store final parent distances:
        for (DistanceEntry<E> ent : assignments.getFirstAssignments()) {
            final E e = ent.getEntry();
            e.setParentDistance(ent.getDistance());
            entries1.add(e);
        }
        for (DistanceEntry<E> ent : assignments.getSecondAssignments()) {
            final E e = ent.getEntry();
            e.setParentDistance(ent.getDistance());
            entries2.add(e);
        }
        node.splitTo(newNode, entries1, entries2);
        // write changes to file
        writeNode(node);
        writeNode(newNode);
        if (log.isDebuggingFine()) {
            log.debugFine(// 
            new StringBuilder(1000).append("Split Node ").append(node.getPageID()).append(" (").append(this.getClass()).append(')').append(// 
            FormatUtil.NEWLINE).append("      newNode ").append(newNode.getPageID()).append(// 
            FormatUtil.NEWLINE).append("      firstPromoted ").append(assignments.getFirstRoutingObject()).append(// 
            FormatUtil.NEWLINE).append("      firstAssignments(").append(node.getPageID()).append(") ").append(assignments.getFirstAssignments()).append(// 
            FormatUtil.NEWLINE).append("      firstCR ").append(assignments.computeFirstCover(node.isLeaf())).append(// 
            FormatUtil.NEWLINE).append("      secondPromoted ").append(assignments.getSecondRoutingObject()).append(// 
            FormatUtil.NEWLINE).append("      secondAssignments(").append(newNode.getPageID()).append(") ").append(assignments.getSecondAssignments()).append(// 
            FormatUtil.NEWLINE).append("      secondCR ").append(assignments.computeSecondCover(node.isLeaf())).append(FormatUtil.NEWLINE));
        }
        // if root was split: create a new root that points the two split nodes
        if (isRoot(node)) {
            // FIXME: stimmen die parentDistance der Kinder in node & splitNode?
            IndexTreePath<E> newRootPath = createNewRoot(node, newNode, assignments.getFirstRoutingObject(), assignments.getSecondRoutingObject());
            adjustTree(newRootPath);
        } else // node is not root
        {
            // get the parent and add the new split node
            E parentEntry = subtree.getParentPath().getEntry();
            N parent = getNode(parentEntry);
            if (log.isDebugging()) {
                log.debugFine("parent " + parent);
            }
            double parentDistance2 = distance(parentEntry.getRoutingObjectID(), assignments.getSecondRoutingObject());
            // logger.warning("parent: "+parent.toString()+" split: " +
            // splitNode.toString()+ " dist:"+parentDistance2);
            parent.addDirectoryEntry(createNewDirectoryEntry(newNode, assignments.getSecondRoutingObject(), parentDistance2));
            // adjust the entry representing the (old) node, that has been split
            double parentDistance1 = distance(parentEntry.getRoutingObjectID(), assignments.getFirstRoutingObject());
            // logger.warning("parent: "+parent.toString()+" node: " +
            // node.toString()+ " dist:"+parentDistance1);
            node.adjustEntry(parent.getEntry(nodeIndex), assignments.getFirstRoutingObject(), parentDistance1, this);
            // write changes in parent to file
            writeNode(parent);
            adjustTree(subtree.getParentPath());
        }
    } else // no overflow, only adjust parameters of the entry representing the node
    {
        if (!isRoot(node)) {
            E parentEntry = subtree.getParentPath().getEntry();
            N parent = getNode(parentEntry);
            E entry = parent.getEntry(subtree.getIndex());
            boolean changed = node.adjustEntry(entry, entry.getRoutingObjectID(), entry.getParentDistance(), this);
            // write changes in parent to file
            if (changed) {
                writeNode(parent);
                adjustTree(subtree.getParentPath());
            }
        } else // root level is reached
        {
            E rootEntry = getRootEntry();
            node.adjustEntry(rootEntry, rootEntry.getRoutingObjectID(), rootEntry.getParentDistance(), this);
        }
    }
}

Example 4 with Logging

use of de.lmu.ifi.dbs.elki.logging.Logging in project elki by elki-project.

the class AbstractMTree method logStatistics.

@Override
public void logStatistics() {
    super.logStatistics();
    Logging log = getLogger();
    if (log.isStatistics()) {
        log.statistics(new LongStatistic(this.getClass().getName() + ".height", getHeight()));
        statistics.logStatistics();
    }
}

Example 5 with Logging

use of de.lmu.ifi.dbs.elki.logging.Logging in project elki by elki-project.

the class SpatialApproximationMaterializeKNNPreprocessor method preprocess.

@Override
protected void preprocess() {
    DistanceQuery<O> distanceQuery = relation.getDistanceQuery(distanceFunction);
    SpatialIndexTree<N, E> index = getSpatialIndex(relation);
    storage = DataStoreUtil.makeStorage(relation.getDBIDs(), DataStoreFactory.HINT_STATIC, KNNList.class);
    MeanVariance pagesize = new MeanVariance();
    MeanVariance ksize = new MeanVariance();
    final Logging log = getLogger();
    if (log.isVerbose()) {
        log.verbose("Approximating nearest neighbor lists to database objects");
    }
    List<E> leaves = index.getLeaves();
    FiniteProgress progress = log.isVerbose() ? new FiniteProgress("Processing leaf nodes", leaves.size(), log) : null;
    for (E leaf : leaves) {
        N node = index.getNode(leaf);
        int size = node.getNumEntries();
        pagesize.put(size);
        if (log.isDebuggingFinest()) {
            log.debugFinest("NumEntires = " + size);
        }
        // Collect the ids in this node.
        ArrayModifiableDBIDs ids = DBIDUtil.newArray(size);
        for (int i = 0; i < size; i++) {
            ids.add(((LeafEntry) node.getEntry(i)).getDBID());
        }
        Object2DoubleOpenHashMap<DBIDPair> cache = new Object2DoubleOpenHashMap<>((size * size * 3) >> 3);
        for (DBIDIter id = ids.iter(); id.valid(); id.advance()) {
            KNNHeap kNN = DBIDUtil.newHeap(k);
            for (DBIDIter id2 = ids.iter(); id2.valid(); id2.advance()) {
                DBIDPair key = DBIDUtil.newPair(id, id2);
                double d = cache.removeDouble(key);
                if (d == d) {
                    // Not NaN
                    // consume the previous result.
                    kNN.insert(d, id2);
                } else {
                    // compute new and store the previous result.
                    d = distanceQuery.distance(id, id2);
                    kNN.insert(d, id2);
                    // put it into the cache, but with the keys reversed
                    key = DBIDUtil.newPair(id2, id);
                    cache.put(key, d);
                }
            }
            ksize.put(kNN.size());
            storage.put(id, kNN.toKNNList());
        }
        if (log.isDebugging() && cache.size() > 0) {
            log.warning("Cache should be empty after each run, but still has " + cache.size() + " elements.");
        }
        log.incrementProcessed(progress);
    }
    log.ensureCompleted(progress);
    if (log.isVerbose()) {
        log.verbose("Average page size = " + pagesize.getMean() + " +- " + pagesize.getSampleStddev());
        log.verbose("On average, " + ksize.getMean() + " +- " + ksize.getSampleStddev() + " neighbors returned.");
    }
}