Examples with DocumentStoreSplitter org.apache.jackrabbit.oak.plugins.document.mongo.DocumentStoreSplitter used on opensource projects

Example 1 with DocumentStoreSplitter

use of org.apache.jackrabbit.oak.plugins.document.mongo.DocumentStoreSplitter in project jackrabbit-oak by apache.

the class DocumentStoreIndexerBase method buildFlatFileStore.

private FlatFileStore buildFlatFileStore(NodeState checkpointedState, CompositeIndexer indexer, Predicate<String> pathPredicate, Set<String> preferredPathElements) throws IOException {
    Stopwatch flatFileStoreWatch = Stopwatch.createStarted();
    int executionCount = 1;
    CompositeException lastException = null;
    List<File> previousDownloadDirs = new ArrayList<>();
    FlatFileStore flatFileStore = null;
    // TODO How to ensure we can safely read from secondary
    DocumentNodeState rootDocumentState = (DocumentNodeState) checkpointedState;
    DocumentNodeStore nodeStore = (DocumentNodeStore) indexHelper.getNodeStore();
    DocumentStoreSplitter splitter = new DocumentStoreSplitter(getMongoDocumentStore());
    List<Long> lastModifiedBreakPoints = splitter.split(Collection.NODES, 0L, 10);
    FlatFileNodeStoreBuilder builder = null;
    int backOffTimeInMillis = 5000;
    MemoryManager memoryManager = new DefaultMemoryManager();
    while (flatFileStore == null && executionCount <= MAX_DOWNLOAD_ATTEMPTS) {
        try {
            builder = new FlatFileNodeStoreBuilder(indexHelper.getWorkDir(), memoryManager).withLastModifiedBreakPoints(lastModifiedBreakPoints).withBlobStore(indexHelper.getGCBlobStore()).withPreferredPathElements((preferredPathElements != null) ? preferredPathElements : indexer.getRelativeIndexedNodeNames()).addExistingDataDumpDir(indexerSupport.getExistingDataDumpDir()).withPathPredicate(pathPredicate).withNodeStateEntryTraverserFactory(new MongoNodeStateEntryTraverserFactory(rootDocumentState.getRootRevision(), nodeStore, getMongoDocumentStore(), traversalLog, indexer));
            for (File dir : previousDownloadDirs) {
                builder.addExistingDataDumpDir(dir);
            }
            flatFileStore = builder.build();
            closer.register(flatFileStore);
        } catch (CompositeException e) {
            e.logAllExceptions("Underlying throwable caught during download", log);
            log.info("Could not build flat file store. Execution count {}. Retries left {}. Time elapsed {}", executionCount, MAX_DOWNLOAD_ATTEMPTS - executionCount, flatFileStoreWatch);
            lastException = e;
            previousDownloadDirs.add(builder.getFlatFileStoreDir());
            if (executionCount < MAX_DOWNLOAD_ATTEMPTS) {
                try {
                    log.info("Waiting for {} millis before retrying", backOffTimeInMillis);
                    Thread.sleep(backOffTimeInMillis);
                    backOffTimeInMillis *= 2;
                } catch (InterruptedException ie) {
                    log.error("Interrupted while waiting before retrying download ", ie);
                }
            }
        }
        executionCount++;
    }
    if (flatFileStore == null) {
        throw new IOException("Could not build flat file store", lastException);
    }
    log.info("Completed the flat file store build in {}", flatFileStoreWatch);
    return flatFileStore;
}

Example 2 with DocumentStoreSplitter

use of org.apache.jackrabbit.oak.plugins.document.mongo.DocumentStoreSplitter in project jackrabbit-oak by apache.

the class DocumentStoreIndexerBase method buildFlatFileStore.

private FlatFileStore buildFlatFileStore(NodeState checkpointedState, CompositeIndexer indexer) throws IOException {
    Stopwatch flatFileStoreWatch = Stopwatch.createStarted();
    int executionCount = 1;
    CompositeException lastException = null;
    List<File> previousDownloadDirs = new ArrayList<>();
    FlatFileStore flatFileStore = null;
    // TODO How to ensure we can safely read from secondary
    DocumentNodeState rootDocumentState = (DocumentNodeState) checkpointedState;
    DocumentNodeStore nodeStore = (DocumentNodeStore) indexHelper.getNodeStore();
    DocumentStoreSplitter splitter = new DocumentStoreSplitter(getMongoDocumentStore());
    List<Long> lastModifiedBreakPoints = splitter.split(Collection.NODES, 0L, 10);
    FlatFileNodeStoreBuilder builder = null;
    int backOffTimeInMillis = 5000;
    MemoryManager memoryManager = new DefaultMemoryManager();
    while (flatFileStore == null && executionCount <= MAX_DOWNLOAD_ATTEMPTS) {
        try {
            builder = new FlatFileNodeStoreBuilder(indexHelper.getWorkDir(), memoryManager).withLastModifiedBreakPoints(lastModifiedBreakPoints).withBlobStore(indexHelper.getGCBlobStore()).withPreferredPathElements(indexer.getRelativeIndexedNodeNames()).addExistingDataDumpDir(indexerSupport.getExistingDataDumpDir()).withNodeStateEntryTraverserFactory(new MongoNodeStateEntryTraverserFactory(rootDocumentState.getRootRevision(), nodeStore, getMongoDocumentStore(), traversalLog, indexer));
            for (File dir : previousDownloadDirs) {
                builder.addExistingDataDumpDir(dir);
            }
            flatFileStore = builder.build();
            closer.register(flatFileStore);
        } catch (CompositeException e) {
            e.logAllExceptions("Underlying throwable caught during download", log);
            log.info("Could not build flat file store. Execution count {}. Retries left {}. Time elapsed {}", executionCount, MAX_DOWNLOAD_ATTEMPTS - executionCount, flatFileStoreWatch);
            lastException = e;
            previousDownloadDirs.add(builder.getFlatFileStoreDir());
            if (executionCount < MAX_DOWNLOAD_ATTEMPTS) {
                try {
                    log.info("Waiting for {} millis before retrying", backOffTimeInMillis);
                    Thread.sleep(backOffTimeInMillis);
                    backOffTimeInMillis *= 2;
                } catch (InterruptedException ie) {
                    log.error("Interrupted while waiting before retrying download ", ie);
                }
            }
        }
        executionCount++;
    }
    if (flatFileStore == null) {
        throw new IOException("Could not build flat file store", lastException);
    }
    log.info("Completed the flat file store build in {}", flatFileStoreWatch);
    return flatFileStore;
}