Examples with TupleIndex org.apache.jena.tdb2.store.tupletable.TupleIndex used on opensource projects

Example 1 with TupleIndex

use of org.apache.jena.tdb2.store.tupletable.TupleIndex in project jena by apache.

the class LoaderOps method copyIndex.

/**
 * Copy a stream to several indexes (sequential version)
 */
public static void copyIndex(Iterator<Tuple<NodeId>> srcIter, TupleIndex[] destIndexes, ProgressMonitor monitor) {
    long counter = 0;
    for (; srcIter.hasNext(); ) {
        counter++;
        Tuple<NodeId> tuple = srcIter.next();
        monitor.tick();
        for (TupleIndex destIdx : destIndexes) {
            if (destIdx != null)
                destIdx.add(tuple);
        }
    }
}

Example 2 with TupleIndex

use of org.apache.jena.tdb2.store.tupletable.TupleIndex in project jena by apache.

the class LoaderOps method idxBTree.

/**
 * Get the BPlusTree index for a {@Link TupleIndex}
 */
public static BPlusTree idxBTree(TupleIndex idx) {
    TupleIndexRecord idxr = (TupleIndexRecord) idx;
    RangeIndex rIndex = idxr.getRangeIndex();
    BPlusTree bpt = (BPlusTree) rIndex;
    return bpt;
}

Example 3 with TupleIndex

use of org.apache.jena.tdb2.store.tupletable.TupleIndex in project jena by apache.

the class Indexer method startBulk.

/**
 * Start the threads that will do the indexing
 */
@Override
public void startBulk() {
    for (int i = 0; i < N; i++) {
        TupleIndex idx = indexes[i];
        BlockingQueue<List<Tuple<NodeId>>> pipe = pipesTripleIndexers[i];
        new Thread(() -> stageIndex(pipe, idx)).start();
    }
}

Example 4 with TupleIndex

use of org.apache.jena.tdb2.store.tupletable.TupleIndex in project jena by apache.

the class PhasedOps method replay.

/**
 * Return (Number, Time in ms)
 */
static ReplayResult replay(TupleIndex srcIdx, Destination<Tuple<NodeId>> dest, MonitorOutput output) {
    ProgressMonitor monitor = ProgressMonitorFactory.progressMonitor("Index", output, LoaderMain.IndexTickPoint, LoaderMain.IndexSuperTick);
    List<Tuple<NodeId>> block = null;
    int len = srcIdx.getTupleLength();
    monitor.start();
    Iterator<Tuple<NodeId>> iter = srcIdx.all();
    while (iter.hasNext()) {
        if (block == null)
            block = new ArrayList<>(LoaderConst.ChunkSize);
        Tuple<NodeId> row = iter.next();
        block.add(row);
        monitor.tick();
        if (block.size() == LoaderConst.ChunkSize) {
            dest.deliver(block);
            block = null;
        }
    }
    if (block != null)
        dest.deliver(block);
    dest.deliver(Collections.emptyList());
    monitor.finish();
    // monitor.finishMessage("Tuples["+len+"]");
    return new ReplayResult(monitor.getTicks(), monitor.getTime());
}

Example 5 with TupleIndex

use of org.apache.jena.tdb2.store.tupletable.TupleIndex in project jena by apache.

the class LoaderMain method executeData.

/**
 * Create data ingestion and primary index building of a {@link LoaderPlan}.
 * In phase 1, separate threads for parsing, node table loading and primary index building,
 *
 * Used by {@link InputStage#MULTI}.
 */
private static StreamRDFCounting executeData(LoaderPlan loaderPlan, DatasetGraphTDB dsgtdb, Map<String, TupleIndex> indexMap, List<BulkStartFinish> dataProcess, MonitorOutput output) {
    StoragePrefixesTDB dps = (StoragePrefixesTDB) dsgtdb.getStoragePrefixes();
    PrefixHandlerBulk prefixHandler = new PrefixHandlerBulk(dps, output);
    dataProcess.add(prefixHandler);
    // -- Phase 2 block. Indexer and Destination (blocks of Tuple<NodeId>)
    TupleIndex[] idx3 = PhasedOps.indexSetFromNames(loaderPlan.primaryLoad3(), indexMap);
    Indexer indexer3 = new Indexer(output, idx3);
    TupleIndex[] idx4 = PhasedOps.indexSetFromNames(loaderPlan.primaryLoad4(), indexMap);
    Indexer indexer4 = new Indexer(output, idx4);
    dataProcess.add(indexer3);
    dataProcess.add(indexer4);
    Destination<Tuple<NodeId>> functionIndexer3 = indexer3.index();
    Destination<Tuple<NodeId>> functionIndexer4 = indexer4.index();
    // -- Phase 2 block.
    // -- Phase 1.
    // This is the other way round to AsyncParser.
    // Here, we return a StreamRDF to pump data into and the rest of the
    // processing is on other threads. AsyncParser has the processing on the caller thread
    // and so the current thread continues when the processing from the parser is finished.
    DataToTuples dtt = new DataToTuples(dsgtdb, functionIndexer3, functionIndexer4, output);
    DataBatcher dataBatcher = new DataBatcher(dtt.data(), prefixHandler.handler(), output);
    dataProcess.add(dtt);
    dataProcess.add(dataBatcher);
    return dataBatcher;
}