Examples with WarmerConfig com.yelp.nrtsearch.server.luceneserver.warming.WarmerConfig used on opensource projects

Example 1 with WarmerConfig

use of com.yelp.nrtsearch.server.luceneserver.warming.WarmerConfig in project nrtsearch by Yelp.

the class ShardState method startReplica.

/**
 * Start this index as replica, pulling NRT changes from the specified primary
 */
public synchronized void startReplica(ReplicationServerClient primaryAddress, long primaryGen, Path dataPath) throws Exception {
    if (isStarted()) {
        throw new IllegalStateException("index \"" + name + "\" was already started");
    }
    // use that to load indexes and other state (registeredFields, settings)
    if (!doCreate && dataPath != null) {
        if (indexState.rootDir != null) {
            synchronized (this) {
                // copy downloaded data into rootDir
                indexState.restoreDir(dataPath, indexState.rootDir);
            }
            indexState.initSaveLoadState();
        }
    }
    // nocommit share code better w/ start and startPrimary!
    try {
        if (indexState.saveLoadState == null) {
            indexState.initSaveLoadState();
        }
        Path indexDirFile;
        if (rootDir == null) {
            indexDirFile = null;
        } else {
            indexDirFile = rootDir.resolve("index");
        }
        origIndexDir = indexState.df.open(indexDirFile, indexState.globalState.getConfiguration().getPreloadConfig());
        // nocommit remove NRTCachingDir too?
        if ((origIndexDir instanceof MMapDirectory) == false) {
            double maxMergeSizeMB = indexState.getDoubleSetting("nrtCachingDirectoryMaxMergeSizeMB", 5.0);
            double maxSizeMB = indexState.getDoubleSetting("nrtCachingDirectoryMaxSizeMB", 60.0);
            if (maxMergeSizeMB > 0 && maxSizeMB > 0) {
                indexDir = new NRTCachingDirectory(origIndexDir, maxMergeSizeMB, maxSizeMB);
            } else {
                indexDir = origIndexDir;
            }
        } else {
            indexDir = origIndexDir;
        }
        manager = null;
        nrtPrimaryNode = null;
        boolean verbose = indexState.globalState.getConfiguration().getIndexVerbose();
        HostPort hostPort = new HostPort(indexState.globalState.getHostName(), indexState.globalState.getReplicationPort());
        nrtReplicaNode = new NRTReplicaNode(indexState.name, primaryAddress, hostPort, REPLICA_ID, indexDir, new ShardSearcherFactory(true, false), verbose ? System.out : new PrintStream(OutputStream.nullOutputStream()), primaryGen, indexState.globalState.getConfiguration().getFileCopyConfig().getAckedCopy());
        if (indexState.globalState.getConfiguration().getSyncInitialNrtPoint()) {
            nrtReplicaNode.syncFromCurrentPrimary(INITIAL_SYNC_PRIMARY_WAIT_MS);
        }
        startSearcherPruningThread(indexState.globalState.getShutdownLatch());
        // Necessary so that the replica "hang onto" all versions sent to it, since the version is
        // sent back to the user on writeNRTPoint
        addRefreshListener(new ReferenceManager.RefreshListener() {

            @Override
            public void beforeRefresh() {
            }

            @Override
            public void afterRefresh(boolean didRefresh) throws IOException {
                SearcherTaxonomyManager.SearcherAndTaxonomy current = acquire();
                try {
                    slm.record(current.searcher);
                } finally {
                    release(current);
                }
            }
        });
        keepAlive = new KeepAlive(this);
        new Thread(keepAlive, "KeepAlive").start();
        WarmerConfig warmerConfig = indexState.globalState.getConfiguration().getWarmerConfig();
        if (warmerConfig.isWarmOnStartup() && indexState.getWarmer() != null) {
            indexState.getWarmer().warmFromS3(indexState, warmerConfig.getWarmingParallelism());
        }
        started = true;
    } finally {
        if (!started) {
            IOUtils.closeWhileHandlingException(reopenThread, nrtReplicaNode, writer, taxoWriter, searcherPruningThread, slm, indexDir, taxoDir);
            writer = null;
        }
    }
}

Example 2 with WarmerConfig

use of com.yelp.nrtsearch.server.luceneserver.warming.WarmerConfig in project nrtsearch by Yelp.

the class IndexState method initWarmer.

public void initWarmer(Archiver archiver) {
    LuceneServerConfiguration configuration = globalState.getConfiguration();
    WarmerConfig warmerConfig = configuration.getWarmerConfig();
    if (warmerConfig.isWarmOnStartup() || warmerConfig.getMaxWarmingQueries() > 0) {
        this.warmer = new Warmer(archiver, configuration.getServiceName(), name, warmerConfig.getMaxWarmingQueries());
    }
}