Examples with DocumentQueue org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue used on opensource projects

Example 1 with DocumentQueue

use of org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue in project jackrabbit-oak by apache.

the class LuceneIndexProviderService method registerLocalIndexObserver.

private void registerLocalIndexObserver(BundleContext bundleContext, IndexTracker tracker, Map<String, ?> config) {
    if (!hybridIndex) {
        log.info("Hybrid indexing feature disabled");
        return;
    }
    int queueSize = PropertiesUtil.toInteger(config.get(PROP_HYBRID_QUEUE_SIZE), PROP_HYBRID_QUEUE_SIZE_DEFAULT);
    documentQueue = new DocumentQueue(queueSize, tracker, getExecutorService(), statisticsProvider);
    LocalIndexObserver localIndexObserver = new LocalIndexObserver(documentQueue, statisticsProvider);
    regs.add(bundleContext.registerService(Observer.class.getName(), localIndexObserver, null));
    int observerQueueSize = 1000;
    int builderMaxSize = 5000;
    regs.add(bundleContext.registerService(JournalPropertyService.class.getName(), new LuceneJournalPropertyService(builderMaxSize), null));
    ExternalObserverBuilder builder = new ExternalObserverBuilder(documentQueue, tracker, statisticsProvider, getExecutorService(), observerQueueSize);
    log.info("Configured JournalPropertyBuilder with max size {} and backed by BackgroundObserver " + "with queue size {}", builderMaxSize, observerQueueSize);
    Observer observer = builder.build();
    externalIndexObserver = builder.getBackgroundObserver();
    regs.add(bundleContext.registerService(Observer.class.getName(), observer, null));
    oakRegs.add(registerMBean(whiteboard, BackgroundObserverMBean.class, externalIndexObserver.getMBean(), BackgroundObserverMBean.TYPE, "LuceneExternalIndexObserver queue stats"));
    log.info("Hybrid indexing enabled for configured indexes with queue size of {}", queueSize);
}

Example 2 with DocumentQueue

use of org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue in project archiva by apache.

the class RepositoryFactory method createRepository.

public Repository createRepository() throws IOException, InvalidFileStoreVersionException {
    createExecutor();
    if (SEGMENT_FILE_TYPE == storeType) {
        fileStore = FileStoreBuilder.fileStoreBuilder(repositoryPath.toFile()).build();
        nodeStore = // 
        SegmentNodeStoreBuilders.builder(fileStore).withStatisticsProvider(// 
        StatisticsProvider.NOOP).build();
    } else if (IN_MEMORY_TYPE == storeType) {
        nodeStore = null;
    } else {
        throw new IllegalArgumentException("Store type " + storeType + " not recognized");
    }
    Oak oak = nodeStore == null ? new Oak() : new Oak(nodeStore);
    oak.with(new RepositoryInitializer() {

        @Override
        public void initialize(@Nonnull NodeBuilder root) {
            log.info("Creating index ");
            NodeBuilder lucene = IndexUtils.getOrCreateOakIndex(root).child("lucene");
            lucene.setProperty(JcrConstants.JCR_PRIMARYTYPE, "oak:QueryIndexDefinition", Type.NAME);
            lucene.setProperty("compatVersion", 2);
            lucene.setProperty("type", "lucene");
            // lucene.setProperty("async", "async");
            lucene.setProperty(INCLUDE_PROPERTY_TYPES, ImmutableSet.of("String"), Type.STRINGS);
            // lucene.setProperty("refresh",true);
            lucene.setProperty("async", ImmutableSet.of("async", "sync"), Type.STRINGS);
            NodeBuilder rules = lucene.child("indexRules").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
            rules.setProperty(":childOrder", // 
            ImmutableSet.of(// 
            "archiva:projectVersion", // 
            "archiva:artifact", // 
            "archiva:facet", // 
            "archiva:namespace", // 
            "archiva:project"), Type.STRINGS);
            NodeBuilder allProps = // 
            rules.child("archiva:projectVersion").child(// 
            "properties").setProperty(JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", // 
            Type.NAME).setProperty(":childOrder", ImmutableSet.of("allProps"), // 
            Type.STRINGS).setProperty("indexNodeName", // 
            true).child(// 
            "allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
            allProps.setProperty("name", ".*");
            allProps.setProperty("isRegexp", true);
            allProps.setProperty("nodeScopeIndex", true);
            allProps.setProperty("index", true);
            allProps.setProperty("analyzed", true);
            // allProps.setProperty("propertyIndex",true);
            allProps = // 
            rules.child("archiva:artifact").child(// 
            "properties").setProperty(JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", // 
            Type.NAME).setProperty(":childOrder", ImmutableSet.of("allProps"), // 
            Type.STRINGS).setProperty("indexNodeName", true).child(// 
            "allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
            allProps.setProperty("name", ".*");
            allProps.setProperty("isRegexp", true);
            allProps.setProperty("nodeScopeIndex", true);
            allProps.setProperty("index", true);
            allProps.setProperty("analyzed", true);
            allProps = // 
            rules.child("archiva:facet").child(// 
            "properties").setProperty(JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", // 
            Type.NAME).setProperty(":childOrder", ImmutableSet.of("allProps"), // 
            Type.STRINGS).setProperty("indexNodeName", // 
            true).child(// 
            "allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
            allProps.setProperty("name", ".*");
            allProps.setProperty("isRegexp", true);
            allProps.setProperty("nodeScopeIndex", true);
            allProps.setProperty("index", true);
            allProps.setProperty("analyzed", true);
            allProps = // 
            rules.child("archiva:namespace").child(// 
            "properties").setProperty(JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", // 
            Type.NAME).setProperty(":childOrder", ImmutableSet.of("allProps"), // 
            Type.STRINGS).setProperty("indexNodeName", // 
            true).child(// 
            "allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
            allProps.setProperty("name", ".*");
            allProps.setProperty("isRegexp", true);
            allProps.setProperty("nodeScopeIndex", true);
            allProps.setProperty("index", true);
            allProps.setProperty("analyzed", true);
            allProps = // 
            rules.child("archiva:project").child(// 
            "properties").setProperty(JcrConstants.JCR_PRIMARYTYPE, "nt:unstructured", // 
            Type.NAME).setProperty(":childOrder", ImmutableSet.of("allProps"), // 
            Type.STRINGS).setProperty("indexNodeName", // 
            true).child(// 
            "allProps").setProperty(JcrConstants.JCR_PRIMARYTYPE, JcrConstants.NT_UNSTRUCTURED, Type.NAME);
            allProps.setProperty("name", ".*");
            allProps.setProperty("isRegexp", true);
            allProps.setProperty("nodeScopeIndex", true);
            allProps.setProperty("index", true);
            allProps.setProperty("analyzed", true);
            log.info("Index: {} myIndex {}", lucene, lucene.getChildNode("myIndex"));
            log.info("myIndex {}", lucene.getChildNode("myIndex").getProperties());
        // IndexUtils.createIndexDefinition(  )
        }
    });
    StatisticsProvider statsProvider = StatisticsProvider.NOOP;
    int queueSize = Integer.getInteger("queueSize", 10000);
    Path indexDir = Files.createTempDirectory("archiva_index");
    log.info("Queue Index {}", indexDir.toString());
    IndexCopier indexCopier = new IndexCopier(executorService, indexDir.toFile(), true);
    NRTIndexFactory nrtIndexFactory = new NRTIndexFactory(indexCopier, statsProvider);
    MountInfoProvider mountInfoProvider = Mounts.defaultMountInfoProvider();
    IndexTracker tracker = new IndexTracker(new DefaultIndexReaderFactory(mountInfoProvider, indexCopier), nrtIndexFactory);
    DocumentQueue queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
    LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, statsProvider);
    LuceneIndexProvider provider = new LuceneIndexProvider(tracker);
    // ExternalObserverBuilder builder = new ExternalObserverBuilder(queue, tracker, statsProvider,
    // executorService, queueSize);
    // Observer observer = builder.build();
    // builder.getBackgroundObserver();
    // 
    LuceneIndexEditorProvider editorProvider = new // 
    LuceneIndexEditorProvider(// 
    null, // 
    tracker, // 
    new ExtractedTextCache(0, 0), null, mountInfoProvider);
    editorProvider.setIndexingQueue(queue);
    log.info("Oak: {} with nodeStore {}", oak, nodeStore);
    Jcr jcr = // 
    new Jcr(oak).with(editorProvider).with(// 
    (Observer) provider).with(localIndexObserver).with(// 
    (QueryIndexProvider) provider);
    // .withAsyncIndexing( "async", 5 );
    StopWatch stopWatch = new StopWatch();
    stopWatch.start();
    Repository r = jcr.createRepository();
    stopWatch.stop();
    log.info("time to create jcr repository: {} ms", stopWatch.getTime());
    // }
    return r;
}

Example 3 with DocumentQueue

use of org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue in project jackrabbit-oak by apache.

the class JsonIndexCommand method createLuceneIndexEditorProvider.

private static LuceneIndexEditorProvider createLuceneIndexEditorProvider() {
    LuceneIndexEditorProvider ep = new LuceneIndexEditorProvider();
    ScheduledExecutorService executorService = MoreExecutors.getExitingScheduledExecutorService((ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(5));
    StatisticsProvider statsProvider = StatisticsProvider.NOOP;
    int queueSize = Integer.getInteger("queueSize", 1000);
    IndexTracker tracker = new IndexTracker();
    DocumentQueue queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
    ep.setIndexingQueue(queue);
    return ep;
}

Example 4 with DocumentQueue

use of org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue in project jackrabbit-oak by apache.

the class HybridIndexTest method prepareLuceneIndexer.

private void prepareLuceneIndexer(File workDir) {
    try {
        indexCopierDir = createTemporaryFolderIn(workDir);
        copier = new IndexCopier(executorService, indexCopierDir, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE, TimeUnit.MILLISECONDS.toSeconds(refreshDeltaMillis), StatisticsProvider.NOOP);
    MountInfoProvider mip = Mounts.defaultMountInfoProvider();
    LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
    IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
    luceneIndexProvider = new LuceneIndexProvider(tracker);
    luceneEditorProvider = new LuceneIndexEditorProvider(copier, tracker, //extractedTextCache
    null, //augmentorFactory
    null, mip);
    queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
    localIndexObserver = new LocalIndexObserver(queue, statsProvider);
    luceneEditorProvider.setIndexingQueue(queue);
}

Example 5 with DocumentQueue

use of org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue in project jackrabbit-oak by apache.

the class HybridIndexTest method prepareLuceneIndexer.

private void prepareLuceneIndexer(File workDir, NodeStore nodeStore) {
    try {
        indexCopierDir = createTemporaryFolderIn(workDir);
        copier = new IndexCopier(executorService, indexCopierDir, true);
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    IndexPathService indexPathService = new IndexPathServiceImpl(nodeStore);
    AsyncIndexInfoService asyncIndexInfoService = new AsyncIndexInfoServiceImpl(nodeStore);
    nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE, TimeUnit.MILLISECONDS.toSeconds(refreshDeltaMillis), StatisticsProvider.NOOP);
    MountInfoProvider mip = Mounts.defaultMountInfoProvider();
    LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
    IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
    luceneIndexProvider = new LuceneIndexProvider(tracker);
    luceneEditorProvider = new LuceneIndexEditorProvider(copier, tracker, // extractedTextCache
    null, // augmentorFactory
    null, mip);
    queue = new DocumentQueue(queueSize, tracker, executorService, statsProvider);
    localIndexObserver = new LocalIndexObserver(queue, statsProvider);
    luceneEditorProvider.setIndexingQueue(queue);
    if (syncIndexing) {
        PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService, statsProvider);
        regs.add(scheduleWithFixedDelay(whiteboard, cleaner, cleanerIntervalInSecs, true, true));
    }
    Thread.setDefaultUncaughtExceptionHandler((t, e) -> log.warn("Uncaught exception", e));
}