Examples with HostPort com.yelp.nrtsearch.server.utils.HostPort used on opensource projects

Example 1 with HostPort

use of com.yelp.nrtsearch.server.utils.HostPort in project nrtsearch by Yelp.

the class ShardState method startReplica.

/**
 * Start this index as replica, pulling NRT changes from the specified primary
 */
public synchronized void startReplica(ReplicationServerClient primaryAddress, long primaryGen, Path dataPath) throws Exception {
    if (isStarted()) {
        throw new IllegalStateException("index \"" + name + "\" was already started");
    }
    // use that to load indexes and other state (registeredFields, settings)
    if (!doCreate && dataPath != null) {
        if (indexState.rootDir != null) {
            synchronized (this) {
                // copy downloaded data into rootDir
                indexState.restoreDir(dataPath, indexState.rootDir);
            }
            indexState.initSaveLoadState();
        }
    }
    // nocommit share code better w/ start and startPrimary!
    try {
        if (indexState.saveLoadState == null) {
            indexState.initSaveLoadState();
        }
        Path indexDirFile;
        if (rootDir == null) {
            indexDirFile = null;
        } else {
            indexDirFile = rootDir.resolve("index");
        }
        origIndexDir = indexState.df.open(indexDirFile, indexState.globalState.getConfiguration().getPreloadConfig());
        // nocommit remove NRTCachingDir too?
        if ((origIndexDir instanceof MMapDirectory) == false) {
            double maxMergeSizeMB = indexState.getDoubleSetting("nrtCachingDirectoryMaxMergeSizeMB", 5.0);
            double maxSizeMB = indexState.getDoubleSetting("nrtCachingDirectoryMaxSizeMB", 60.0);
            if (maxMergeSizeMB > 0 && maxSizeMB > 0) {
                indexDir = new NRTCachingDirectory(origIndexDir, maxMergeSizeMB, maxSizeMB);
            } else {
                indexDir = origIndexDir;
            }
        } else {
            indexDir = origIndexDir;
        }
        manager = null;
        nrtPrimaryNode = null;
        boolean verbose = indexState.globalState.getConfiguration().getIndexVerbose();
        HostPort hostPort = new HostPort(indexState.globalState.getHostName(), indexState.globalState.getReplicationPort());
        nrtReplicaNode = new NRTReplicaNode(indexState.name, primaryAddress, hostPort, REPLICA_ID, indexDir, new ShardSearcherFactory(true, false), verbose ? System.out : new PrintStream(OutputStream.nullOutputStream()), primaryGen, indexState.globalState.getConfiguration().getFileCopyConfig().getAckedCopy());
        if (indexState.globalState.getConfiguration().getSyncInitialNrtPoint()) {
            nrtReplicaNode.syncFromCurrentPrimary(INITIAL_SYNC_PRIMARY_WAIT_MS);
        }
        startSearcherPruningThread(indexState.globalState.getShutdownLatch());
        // Necessary so that the replica "hang onto" all versions sent to it, since the version is
        // sent back to the user on writeNRTPoint
        addRefreshListener(new ReferenceManager.RefreshListener() {

            @Override
            public void beforeRefresh() {
            }

            @Override
            public void afterRefresh(boolean didRefresh) throws IOException {
                SearcherTaxonomyManager.SearcherAndTaxonomy current = acquire();
                try {
                    slm.record(current.searcher);
                } finally {
                    release(current);
                }
            }
        });
        keepAlive = new KeepAlive(this);
        new Thread(keepAlive, "KeepAlive").start();
        WarmerConfig warmerConfig = indexState.globalState.getConfiguration().getWarmerConfig();
        if (warmerConfig.isWarmOnStartup() && indexState.getWarmer() != null) {
            indexState.getWarmer().warmFromS3(indexState, warmerConfig.getWarmingParallelism());
        }
        started = true;
    } finally {
        if (!started) {
            IOUtils.closeWhileHandlingException(reopenThread, nrtReplicaNode, writer, taxoWriter, searcherPruningThread, slm, indexDir, taxoDir);
            writer = null;
        }
    }
}

Example 2 with HostPort

use of com.yelp.nrtsearch.server.utils.HostPort in project nrtsearch by Yelp.

the class NRTPrimaryNode method close.

@Override
public void close() throws IOException {
    logger.info("CLOSE NRT PRIMARY");
    Iterator<ReplicaDetails> it = replicasInfos.iterator();
    while (it.hasNext()) {
        ReplicaDetails replicaDetails = it.next();
        ReplicationServerClient replicationServerClient = replicaDetails.getReplicationServerClient();
        HostPort replicaHostPort = replicaDetails.getHostPort();
        logger.info(String.format("CLOSE NRT PRIMARY, closing replica channel host:%s, port:%s", replicaHostPort.getHostName(), replicaHostPort.getPort()));
        replicationServerClient.close();
        it.remove();
    }
    super.close();
}

Example 3 with HostPort

use of com.yelp.nrtsearch.server.utils.HostPort in project nrtsearch by Yelp.

the class GetNodesInfoHandler method handle.

@Override
public GetNodesResponse handle(IndexState indexState, GetNodesRequest getNodesRequest) throws HandlerException {
    GetNodesResponse.Builder builder = GetNodesResponse.newBuilder();
    ShardState shardState = indexState.getShard(0);
    if (!shardState.isPrimary() || !shardState.isStarted()) {
        logger.warn("index \"" + indexState.name + "\" is not a primary or was not started yet");
    } else {
        // shard is a primary and started
        Collection<NRTPrimaryNode.ReplicaDetails> replicasInfo = shardState.nrtPrimaryNode.getNodesInfo();
        for (NRTPrimaryNode.ReplicaDetails replica : replicasInfo) {
            HostPort hostPort = replica.getHostPort();
            builder.addNodes(NodeInfo.newBuilder().setHostname(hostPort.getHostName()).setPort(hostPort.getPort()).build());
        }
    }
    return builder.build();
}

Example 4 with HostPort

use of com.yelp.nrtsearch.server.utils.HostPort in project nrtsearch by Yelp.

the class ShardState method startPrimary.

/**
 * Start this index as primary, to NRT-replicate to replicas. primaryGen should be incremented
 * each time a new primary is promoted for a given index.
 */
public synchronized void startPrimary(long primaryGen, Path dataPath) throws Exception {
    if (isStarted()) {
        throw new IllegalStateException("index \"" + name + "\" was already started");
    }
    try {
        // use that to load indexes and other state (registeredFields, settings)
        if (!doCreate && dataPath != null) {
            if (indexState.rootDir != null) {
                synchronized (this) {
                    // copy downloaded data into rootDir
                    indexState.restoreDir(dataPath, indexState.rootDir);
                }
                indexState.initSaveLoadState();
            }
        }
        if (indexState.saveLoadState == null) {
            indexState.initSaveLoadState();
        }
        Path indexDirFile;
        if (rootDir == null) {
            indexDirFile = null;
        } else {
            indexDirFile = rootDir.resolve("index");
        }
        origIndexDir = indexState.df.open(indexDirFile, indexState.globalState.getConfiguration().getPreloadConfig());
        if ((origIndexDir instanceof MMapDirectory) == false) {
            double maxMergeSizeMB = indexState.getDoubleSetting("nrtCachingDirectoryMaxMergeSizeMB", 5.0);
            double maxSizeMB = indexState.getDoubleSetting("nrtCachingDirectoryMaxSizeMB", 60.0);
            if (maxMergeSizeMB > 0 && maxSizeMB > 0) {
                indexDir = new NRTCachingDirectory(origIndexDir, maxMergeSizeMB, maxSizeMB);
            } else {
                indexDir = origIndexDir;
            }
        } else {
            indexDir = origIndexDir;
        }
        // Rather than rely on IndexWriter/TaxonomyWriter to
        // figure out if an index is new or not by passing
        // CREATE_OR_APPEND (which can be dangerous), we
        // already know the intention from the app (whether
        // it called createIndex vs openIndex), so we make it
        // explicit here:
        IndexWriterConfig.OpenMode openMode;
        if (doCreate) {
            // nocommit shouldn't we set doCreate=false after we've done the create?
            openMode = IndexWriterConfig.OpenMode.CREATE;
        } else {
            openMode = IndexWriterConfig.OpenMode.APPEND;
        }
        // TODO: get facets working!
        boolean verbose = indexState.globalState.getConfiguration().getIndexVerbose();
        writer = new IndexWriter(indexDir, indexState.getIndexWriterConfig(openMode, origIndexDir, shardOrd));
        LiveIndexWriterConfig writerConfig = writer.getConfig();
        MergePolicy mergePolicy = writerConfig.getMergePolicy();
        // Disable merges while NrtPrimaryNode isn't initalized (ISSUE-210)
        writerConfig.setMergePolicy(NoMergePolicy.INSTANCE);
        snapshots = (PersistentSnapshotDeletionPolicy) writerConfig.getIndexDeletionPolicy();
        // loads its commits after writer calls .onInit:
        for (IndexCommit c : snapshots.getSnapshots()) {
            long gen = c.getGeneration();
            SegmentInfos sis = SegmentInfos.readCommit(origIndexDir, IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
            snapshotGenToVersion.put(c.getGeneration(), sis.getVersion());
        }
        HostPort hostPort = new HostPort(indexState.globalState.getHostName(), indexState.globalState.getReplicationPort());
        nrtPrimaryNode = new NRTPrimaryNode(indexState.name, hostPort, writer, 0, primaryGen, -1, new ShardSearcherFactory(false, true), verbose ? System.out : new PrintStream(OutputStream.nullOutputStream()));
        // Enable merges
        writerConfig.setMergePolicy(mergePolicy);
        // nocommit this isn't used?
        searcherManager = new NRTPrimaryNode.PrimaryNodeReferenceManager(nrtPrimaryNode, new SearcherFactory() {

            @Override
            public IndexSearcher newSearcher(IndexReader r, IndexReader previousReader) throws IOException {
                IndexSearcher searcher = new MyIndexSearcher(r, new MyIndexSearcher.ExecutorWithParams(searchExecutor, indexState.getSliceMaxDocs(), indexState.getSliceMaxSegments(), indexState.getVirtualShards()));
                searcher.setSimilarity(indexState.sim);
                return searcher;
            }
        });
        restartReopenThread();
        startSearcherPruningThread(indexState.globalState.getShutdownLatch());
        started = true;
    } finally {
        if (!started) {
            IOUtils.closeWhileHandlingException(reopenThread, nrtPrimaryNode, writer, taxoWriter, searcherPruningThread, slm, indexDir, taxoDir);
            writer = null;
        }
    }
}