Examples with IndexCopier org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier used on opensource projects

Example 11 with IndexCopier

use of org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier in project jackrabbit-oak by apache.

the class HybridIndexTest method prepareLuceneIndexer.

private void prepareLuceneIndexer(File workDir) {
    try {
        indexCopierDir = createTemporaryFolderIn(workDir);
        copier = new IndexCopier(executorService, indexCopierDir, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE, TimeUnit.MILLISECONDS.toSeconds(refreshDeltaMillis), StatisticsProvider.NOOP);
    MountInfoProvider mip = Mounts.defaultMountInfoProvider();
    LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
    IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
    luceneIndexProvider = new LuceneIndexProvider(tracker);
    luceneEditorProvider = new LuceneIndexEditorProvider(copier, tracker, //extractedTextCache
    null, //augmentorFactory
    null, mip);
    queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
    localIndexObserver = new LocalIndexObserver(queue, statsProvider);
    luceneEditorProvider.setIndexingQueue(queue);
}

Example 12 with IndexCopier

use of org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier in project jackrabbit-oak by apache.

the class HybridIndexTest method createRepository.

@Override
protected ContentRepository createRepository() {
    IndexCopier copier;
    try {
        copier = new IndexCopier(executorService, temporaryFolder.getRoot());
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    MountInfoProvider mip = defaultMountInfoProvider();
    nrtIndexFactory = new NRTIndexFactory(copier, clock, TimeUnit.MILLISECONDS.toSeconds(refreshDelta), StatisticsProvider.NOOP);
    nrtIndexFactory.setAssertAllResourcesClosed(true);
    LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
    IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
    luceneIndexProvider = new LuceneIndexProvider(tracker);
    queue = new DocumentQueue(100, tracker, sameThreadExecutor());
    LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(copier, tracker, null, null, mip);
    editorProvider.setIndexingQueue(queue);
    LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, StatisticsProvider.NOOP);
    nodeStore = new MemoryNodeStore();
    Oak oak = new Oak(nodeStore).with(new InitialContent()).with(new OpenSecurityProvider()).with((QueryIndexProvider) luceneIndexProvider).with((Observer) luceneIndexProvider).with(localIndexObserver).with(editorProvider).with(new PropertyIndexEditorProvider()).with(new NodeTypeIndexProvider()).with(optionalEditorProvider).with(new NodeCounterEditorProvider()).withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1));
    wb = oak.getWhiteboard();
    return oak.createContentRepository();
}

Example 13 with IndexCopier

use of org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier in project jackrabbit-oak by apache.

the class NRTIndexTest method setUp.

@Before
public void setUp() throws IOException {
    indexCopier = new IndexCopier(sameThreadExecutor(), temporaryFolder.getRoot());
    indexFactory = new NRTIndexFactory(indexCopier, StatisticsProvider.NOOP);
    indexFactory.setAssertAllResourcesClosed(true);
    LuceneIndexEditorContext.configureUniqueId(builder);
}

Example 14 with IndexCopier

use of org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier in project jackrabbit-oak by apache.

the class DocumentQueueTest method benchMarkIndexWriter.

// @Test
public void benchMarkIndexWriter() throws Exception {
    Executor executor = Executors.newFixedThreadPool(5);
    IndexCopier indexCopier = new IndexCopier(executor, temporaryFolder.getRoot());
    indexFactory = new NRTIndexFactory(indexCopier, clock, TimeUnit.MILLISECONDS.toSeconds(refreshDelta), StatisticsProvider.NOOP);
    tracker = new IndexTracker(new DefaultIndexReaderFactory(defaultMountInfoProvider(), indexCopier), indexFactory);
    NodeState indexed = createAndPopulateAsyncIndex(IndexingMode.NRT);
    tracker.update(indexed);
    DocumentQueue queue = new DocumentQueue(1000, tracker, executor);
    /*
            Sample output
            [nrt] Time taken for 10000 is 639.3 ms with waits 1
            [sync] Time taken for 10000 is 30.34 s

            Refreshing reader after every commit would slow down things
         */
    LuceneDoc doc = createDoc("/a/b", "a");
    int numDocs = 10000;
    Stopwatch w = Stopwatch.createStarted();
    int waitCount = 0;
    for (int i = 0; i < numDocs; i++) {
        while (!queue.add(doc)) {
            waitCount++;
        }
    }
    System.out.printf("%n[nrt] Time taken for %d is %s with waits %d%n", numDocs, w, waitCount);
    indexed = createAndPopulateAsyncIndex(IndexingMode.SYNC);
    tracker.update(indexed);
    queue = new DocumentQueue(1000, tracker, executor);
    w = Stopwatch.createStarted();
    for (int i = 0; i < numDocs; i++) {
        ListMultimap<String, LuceneDoc> docs = ArrayListMultimap.create();
        docs.get("/oak:index/fooIndex").add(doc);
        queue.addAllSynchronously(docs.asMap());
    }
    System.out.printf("%n[sync] Time taken for %d is %s%n", numDocs, w);
}

Example 15 with IndexCopier

use of org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier in project jackrabbit-oak by apache.

the class HybridIndexTest method prepareLuceneIndexer.

private void prepareLuceneIndexer(File workDir, NodeStore nodeStore) {
    try {
        indexCopierDir = createTemporaryFolderIn(workDir);
        copier = new IndexCopier(executorService, indexCopierDir, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    IndexPathService indexPathService = new IndexPathServiceImpl(nodeStore);
    AsyncIndexInfoService asyncIndexInfoService = new AsyncIndexInfoServiceImpl(nodeStore);
    nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE, TimeUnit.MILLISECONDS.toSeconds(refreshDeltaMillis), StatisticsProvider.NOOP);
    MountInfoProvider mip = Mounts.defaultMountInfoProvider();
    LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
    IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
    luceneIndexProvider = new LuceneIndexProvider(tracker);
    luceneEditorProvider = new LuceneIndexEditorProvider(copier, tracker, // extractedTextCache
    null, // augmentorFactory
    null, mip);
    queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
    localIndexObserver = new LocalIndexObserver(queue, statsProvider);
    luceneEditorProvider.setIndexingQueue(queue);
    if (syncIndexing) {
        PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService, statsProvider);
        regs.add(scheduleWithFixedDelay(whiteboard, cleaner, cleanerIntervalInSecs, true, true));
    }
    Thread.setDefaultUncaughtExceptionHandler((t, e) -> log.warn("Uncaught exception", e));
}