Examples with ArrayDBIDs de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs used on opensource projects

Example 1 with ArrayDBIDs

use of de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs in project elki by elki-project.

the class ParallelExecutor method run.

/**
 * Run a task on all available CPUs.
 *
 * @param ids IDs to process
 * @param procs Processors to run
 */
public static void run(DBIDs ids, Processor... procs) {
    ParallelCore core = ParallelCore.getCore();
    core.connect();
    try {
        // TODO: try different strategies anyway!
        ArrayDBIDs aids = DBIDUtil.ensureArray(ids);
        final int size = aids.size();
        int numparts = core.getParallelism();
        // TODO: are there better heuristics for choosing this?
        numparts = (size > numparts * numparts * 16) ? numparts * Math.max(1, numparts - 1) : numparts;
        final int blocksize = (size + (numparts - 1)) / numparts;
        List<Future<ArrayDBIDs>> parts = new ArrayList<>(numparts);
        for (int i = 0; i < numparts; i++) {
            final int start = i * blocksize;
            final int end = Math.min(start + blocksize, size);
            Callable<ArrayDBIDs> run = new BlockArrayRunner(aids, start, end, procs);
            parts.add(core.submit(run));
        }
        for (Future<ArrayDBIDs> fut : parts) {
            fut.get();
        }
    } catch (ExecutionException e) {
        throw new RuntimeException("Processor execution failed.", e);
    } catch (InterruptedException e) {
        throw new RuntimeException("Parallel execution interrupted.");
    } finally {
        core.disconnect();
    }
}

Example 2 with ArrayDBIDs

use of de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs in project elki by elki-project.

the class SLINKHDBSCANLinearMemory method run.

/**
 * Run the algorithm
 *
 * @param db Database
 * @param relation Relation
 * @return Clustering hierarchy
 */
public PointerDensityHierarchyRepresentationResult run(Database db, Relation<O> relation) {
    final DistanceQuery<O> distQ = db.getDistanceQuery(relation, getDistanceFunction());
    final KNNQuery<O> knnQ = db.getKNNQuery(distQ, minPts);
    // We need array addressing later.
    final ArrayDBIDs ids = DBIDUtil.ensureArray(relation.getDBIDs());
    // Compute the core distances
    // minPts + 1: ignore query point.
    final WritableDoubleDataStore coredists = computeCoreDists(ids, knnQ, minPts);
    WritableDBIDDataStore pi = DataStoreUtil.makeDBIDStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_STATIC);
    WritableDoubleDataStore lambda = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_STATIC, Double.POSITIVE_INFINITY);
    // Temporary storage for m.
    WritableDoubleDataStore m = DataStoreUtil.makeDoubleStorage(ids, DataStoreFactory.HINT_HOT | DataStoreFactory.HINT_TEMP);
    FiniteProgress progress = LOG.isVerbose() ? new FiniteProgress("Running HDBSCAN*-SLINK", ids.size(), LOG) : null;
    // has to be an array for monotonicity reasons!
    ModifiableDBIDs processedIDs = DBIDUtil.newArray(ids.size());
    for (DBIDIter id = ids.iter(); id.valid(); id.advance()) {
        // Steps 1,3,4 are exactly as in SLINK
        step1(id, pi, lambda);
        // Step 2 is modified to use a different distance
        step2(id, processedIDs, distQ, coredists, m);
        step3(id, pi, lambda, processedIDs, m);
        step4(id, pi, lambda, processedIDs);
        processedIDs.add(id);
        LOG.incrementProcessed(progress);
    }
    LOG.ensureCompleted(progress);
    return new PointerDensityHierarchyRepresentationResult(ids, pi, lambda, distQ.getDistanceFunction().isSquared(), coredists);
}

Example 3 with ArrayDBIDs

use of de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs in project elki by elki-project.

the class ChengAndChurch method biclustering.

@Override
public Clustering<BiclusterWithInversionsModel> biclustering() {
    double[][] mat = RelationUtil.relationAsMatrix(relation, rowIDs);
    BiclusterCandidate cand = new BiclusterCandidate(getRowDim(), getColDim());
    Clustering<BiclusterWithInversionsModel> result = new Clustering<>("Cheng-and-Church", "Cheng and Church Biclustering");
    ModifiableDBIDs noise = DBIDUtil.newHashSet(relation.getDBIDs());
    FiniteProgress prog = LOG.isVerbose() ? new FiniteProgress("Extracting Cluster", n, LOG) : null;
    for (int i = 0; i < n; i++) {
        cand.reset();
        multipleNodeDeletion(mat, cand);
        if (LOG.isVeryVerbose()) {
            LOG.veryverbose("Residue after Alg 2: " + cand.residue + " " + cand.rowcard + "x" + cand.colcard);
        }
        singleNodeDeletion(mat, cand);
        if (LOG.isVeryVerbose()) {
            LOG.veryverbose("Residue after Alg 1: " + cand.residue + " " + cand.rowcard + "x" + cand.colcard);
        }
        nodeAddition(mat, cand);
        if (LOG.isVeryVerbose()) {
            LOG.veryverbose("Residue after Alg 3: " + cand.residue + " " + cand.rowcard + "x" + cand.colcard);
        }
        cand.maskMatrix(mat, dist);
        BiclusterWithInversionsModel model = new BiclusterWithInversionsModel(colsBitsetToIDs(cand.cols), rowsBitsetToIDs(cand.irow));
        final ArrayDBIDs cids = rowsBitsetToIDs(cand.rows);
        noise.removeDBIDs(cids);
        result.addToplevelCluster(new Cluster<>(cids, model));
        if (LOG.isVerbose()) {
            LOG.verbose("Score of bicluster " + (i + 1) + ": " + cand.residue + "\n");
            LOG.verbose("Number of rows: " + cand.rowcard + "\n");
            LOG.verbose("Number of columns: " + cand.colcard + "\n");
        // LOG.verbose("Total number of masked values: " + maskedVals.size() +
        // "\n");
        }
        LOG.incrementProcessed(prog);
    }
    // Add a noise cluster, full-dimensional.
    if (!noise.isEmpty()) {
        long[] allcols = BitsUtil.ones(getColDim());
        BiclusterWithInversionsModel model = new BiclusterWithInversionsModel(colsBitsetToIDs(allcols), DBIDUtil.EMPTYDBIDS);
        result.addToplevelCluster(new Cluster<>(noise, true, model));
    }
    LOG.ensureCompleted(prog);
    return result;
}

Example 4 with ArrayDBIDs

use of de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs in project elki by elki-project.

the class MaterializeKNNAndRKNNPreprocessor method objectsInserted.

@Override
protected void objectsInserted(DBIDs ids) {
    StepProgress stepprog = getLogger().isVerbose() ? new StepProgress(3) : null;
    ArrayDBIDs aids = DBIDUtil.ensureArray(ids);
    // materialize the new kNNs and RkNNs
    getLogger().beginStep(stepprog, 1, "New insertions ocurred, materialize their new kNNs and RkNNs.");
    materializeKNNAndRKNNs(aids, null);
    // update the old kNNs and RkNNs
    getLogger().beginStep(stepprog, 2, "New insertions ocurred, update the affected kNNs and RkNNs.");
    ArrayDBIDs rkNN_ids = updateKNNsAndRkNNs(ids);
    // inform listener
    getLogger().beginStep(stepprog, 3, "New insertions ocurred, inform listeners.");
    fireKNNsInserted(ids, rkNN_ids);
    getLogger().ensureCompleted(stepprog);
}

Example 5 with ArrayDBIDs

use of de.lmu.ifi.dbs.elki.database.ids.ArrayDBIDs in project elki by elki-project.

the class MaterializeKNNAndRKNNPreprocessor method objectsRemoved.

@Override
protected void objectsRemoved(DBIDs ids) {
    StepProgress stepprog = getLogger().isVerbose() ? new StepProgress(3) : null;
    // For debugging: valid DBIDs still in the database.
    final DBIDs valid = DBIDUtil.ensureSet(distanceQuery.getRelation().getDBIDs());
    ArrayDBIDs aids = DBIDUtil.ensureArray(ids);
    // delete the materialized (old) kNNs and RkNNs
    getLogger().beginStep(stepprog, 1, "New deletions ocurred, remove their materialized kNNs and RkNNs.");
    // Temporary storage of removed lists
    List<KNNList> kNNs = new ArrayList<>(ids.size());
    List<TreeSet<DoubleDBIDPair>> rkNNs = new ArrayList<>(ids.size());
    for (DBIDIter iter = aids.iter(); iter.valid(); iter.advance()) {
        kNNs.add(storage.get(iter));
        for (DBIDIter it = storage.get(iter).iter(); it.valid(); it.advance()) {
            if (!valid.contains(it) && !ids.contains(it)) {
                LOG.warning("False kNN: " + it);
            }
        }
        storage.delete(iter);
        rkNNs.add(materialized_RkNN.get(iter));
        for (DoubleDBIDPair it : materialized_RkNN.get(iter)) {
            if (!valid.contains(it) && !ids.contains(it)) {
                LOG.warning("False RkNN: " + it);
            }
        }
        materialized_RkNN.delete(iter);
    }
    // Keep only those IDs not also removed
    ArrayDBIDs kNN_ids = affectedkNN(kNNs, aids);
    ArrayDBIDs rkNN_ids = affectedRkNN(rkNNs, aids);
    // update the affected kNNs and RkNNs
    getLogger().beginStep(stepprog, 2, "New deletions ocurred, update the affected kNNs and RkNNs.");
    // Recompute the kNN for affected objects (in rkNN lists)
    {
        List<? extends KNNList> kNNList = knnQuery.getKNNForBulkDBIDs(rkNN_ids, k);
        int i = 0;
        for (DBIDIter reknn = rkNN_ids.iter(); reknn.valid(); reknn.advance(), i++) {
            if (kNNList.get(i) == null && !valid.contains(reknn)) {
                LOG.warning("BUG in online kNN/RkNN maintainance: " + DBIDUtil.toString(reknn) + " no longer in database.");
                continue;
            }
            assert (kNNList.get(i) != null);
            storage.put(reknn, kNNList.get(i));
            for (DoubleDBIDListIter it = kNNList.get(i).iter(); it.valid(); it.advance()) {
                materialized_RkNN.get(it).add(makePair(it, reknn));
            }
        }
    }
    // remove objects from RkNNs of objects (in kNN lists)
    {
        SetDBIDs idsSet = DBIDUtil.ensureSet(ids);
        for (DBIDIter nn = kNN_ids.iter(); nn.valid(); nn.advance()) {
            TreeSet<DoubleDBIDPair> rkNN = materialized_RkNN.get(nn);
            for (Iterator<DoubleDBIDPair> it = rkNN.iterator(); it.hasNext(); ) {
                if (idsSet.contains(it.next())) {
                    it.remove();
                }
            }
        }
    }
    // inform listener
    getLogger().beginStep(stepprog, 3, "New deletions ocurred, inform listeners.");
    fireKNNsRemoved(ids, rkNN_ids);
    getLogger().ensureCompleted(stepprog);
}