Examples with SearcherVersion com.yelp.nrtsearch.server.grpc.SearcherVersion used on opensource projects

Example 1 with SearcherVersion

use of com.yelp.nrtsearch.server.grpc.SearcherVersion in project nrtsearch by Yelp.

the class WriteNRTPointCommand method call.

@Override
public Integer call() throws Exception {
    LuceneServerClient client = baseCmd.getClient();
    try {
        ReplicationServerClient replicationServerClient = new ReplicationServerClient(getHostName(), getPort());
        SearcherVersion searcherVersion = replicationServerClient.writeNRTPoint(getIndexName());
        logger.info("didRefresh: " + searcherVersion.getDidRefresh());
        logger.info("searcherVersion: " + searcherVersion.getVersion());
    } finally {
        client.shutdown();
    }
    return 0;
}

Example 2 with SearcherVersion

use of com.yelp.nrtsearch.server.grpc.SearcherVersion in project nrtsearch by Yelp.

the class GetCurrentSearcherVersion method call.

@Override
public Integer call() throws Exception {
    LuceneServerClient client = baseCmd.getClient();
    try {
        ReplicationServerClient replServerClient = new ReplicationServerClient(getHostName(), getPort());
        SearcherVersion searcherVersion = replServerClient.getCurrentSearcherVersion(getIndexName());
        logger.info("searcherVersion: " + searcherVersion.getVersion());
    } finally {
        client.shutdown();
    }
    return 0;
}

Example 3 with SearcherVersion

use of com.yelp.nrtsearch.server.grpc.SearcherVersion in project nrtsearch by Yelp.

the class YelpReviewsTest method main.

public static void main(String[] args) throws IOException, InterruptedException {
    Path yelp_reviews_test_base_path = Paths.get(System.getProperty("user.home"), "lucene", "server", "scratch", "yelp_reviews_test");
    GrpcServer.rmDir(yelp_reviews_test_base_path);
    GrpcServer.rmDir(Paths.get("primary_state"));
    GrpcServer.rmDir(Paths.get("replica_state"));
    GrpcServer.rmDir(Paths.get("primary_index_base"));
    GrpcServer.rmDir(Paths.get("replica_index_base"));
    // create empty primary and secondary dirs
    Path primaryDir = yelp_reviews_test_base_path.resolve("primary");
    Path replicaDir = yelp_reviews_test_base_path.resolve("replica");
    Files.createDirectories(primaryDir);
    Files.createDirectories(replicaDir);
    // create primary and secondary, server and client log files
    String primaryClientCommandLog = primaryDir.resolve(CLIENT_LOG).toString();
    String secondaryClientCommandLog = replicaDir.resolve(CLIENT_LOG).toString();
    logger.info("Temporary directory: {}", yelp_reviews_test_base_path);
    Process primaryServerProcess = startServer(primaryDir.resolve(SERVER_LOG).toString(), getLuceneServerPrimaryConfigurationYaml());
    Process replicaServerProcess = startServer(replicaDir.resolve(SERVER_LOG).toString(), getLuceneServerReplicaConfigurationYaml());
    HostPort primaryHostPort = new HostPort(getLuceneServerPrimaryConfigurationYaml());
    HostPort secondaryHostPort = new HostPort(getLuceneServerReplicaConfigurationYaml());
    LuceneServerClient primaryServerClient = new LuceneServerClient(primaryHostPort.hostName, primaryHostPort.port);
    LuceneServerClient secondaryServerClient = new LuceneServerClient(secondaryHostPort.hostName, secondaryHostPort.port);
    // healthcheck, make sure servers are up
    ensureServersUp(primaryServerClient);
    ensureServersUp(secondaryServerClient);
    CompletableFuture<Process> primaryServer = primaryServerProcess.onExit();
    CompletableFuture<Process> replicaServer = replicaServerProcess.onExit();
    try {
        // create indexes
        createIndex(primaryServerClient);
        createIndex(secondaryServerClient);
        // live settings -- only primary
        liveSettings(primaryServerClient);
        // register
        registerFields(primaryServerClient);
        registerFields(secondaryServerClient);
        // settings
        settings(primaryServerClient, ServerType.primary);
        settings(secondaryServerClient, ServerType.replica);
        // start primary index
        StartIndexRequest startIndexRequest = StartIndexRequest.newBuilder().setIndexName(INDEX_NAME).setMode(Mode.PRIMARY).setPrimaryGen(0).build();
        startIndex(primaryServerClient, startIndexRequest);
        // start replica index
        startIndexRequest = StartIndexRequest.newBuilder().setIndexName(INDEX_NAME).setMode(Mode.REPLICA).setPrimaryAddress(primaryHostPort.hostName).setPort(primaryHostPort.replicationPort).build();
        startIndex(secondaryServerClient, startIndexRequest);
        int availableProcessors = Runtime.getRuntime().availableProcessors();
        int MAX_INDEXING_THREADS = availableProcessors > 8 ? availableProcessors / 4 : availableProcessors;
        int MAX_SEARCH_THREADS = availableProcessors > 8 ? availableProcessors / 4 : availableProcessors;
        AtomicBoolean indexingDone = new AtomicBoolean(false);
        // check search hits on replica - in a separate threadpool
        final ExecutorService searchService = createExecutorService(MAX_SEARCH_THREADS, "LuceneSearch");
        Future<Double> searchFuture = searchService.submit(new SearchTask(secondaryServerClient, indexingDone));
        // index to primary - in a separate threadpool
        final ExecutorService indexService = createExecutorService(MAX_INDEXING_THREADS, "LuceneIndexing");
        Path reviews = Paths.get(System.getProperty("user.home"), "reviews.json");
        if (Files.exists(reviews)) {
            logger.info("Input file {} will be indexed", reviews);
        } else {
            String reviewStr = getPathAsStr("reviews.json", ServerType.unknown);
            logger.warn("Input file {} does not exist using default resource from {}", reviews, reviewStr);
            reviews = Paths.get(reviewStr);
        }
        long t1 = System.nanoTime();
        List<Future<Long>> results = ParallelDocumentIndexer.buildAndIndexDocs(new OneDocBuilderImpl(), reviews, indexService, primaryServerClient);
        // wait till all indexing done and notify search thread once done
        for (Future<Long> each : results) {
            Long genId = each.get();
            logger.info("ParallelDocumentIndexer.buildAndIndexDocs returned genId: {}", genId);
        }
        long t2 = System.nanoTime();
        long timeMilliSecs = (t2 - t1) / (1000 * 1000);
        logger.info("ParallelDocumentIndexer.buildAndIndexDocs took {} milliSecs", timeMilliSecs);
        // stop search now
        logger.info("Signal SearchTask to end");
        indexingDone.set(true);
        logger.info("Search result totalHits: {}", searchFuture.get());
        // publishNRT, get latest searcher version and search over replica again with searcherVersion
        ReplicationServerClient primaryReplicationClient = new ReplicationServerClient(primaryHostPort.hostName, primaryHostPort.replicationPort);
        SearcherVersion searcherVersion = primaryReplicationClient.writeNRTPoint(INDEX_NAME);
        new SearchTask(secondaryServerClient, indexingDone).getSearchTotalHits(searcherVersion.getVersion());
        logger.info("done...");
    } catch (StatusRuntimeException e) {
        logger.error("RPC failed with status {}", e.getStatus());
        throw new RuntimeException(e);
    } catch (ExecutionException e) {
        logger.error("Task launched async failed {}", e.getMessage());
        throw new RuntimeException(e);
    } finally {
        // stop servers
        primaryServer.cancel(true);
        replicaServer.cancel(true);
        primaryServerProcess.destroy();
        replicaServerProcess.destroy();
        logger.info("cleanup done...");
    }
}

Example 4 with SearcherVersion

use of com.yelp.nrtsearch.server.grpc.SearcherVersion in project nrtsearch by Yelp.

the class WriteNRTPointHandler method handle.

@Override
public SearcherVersion handle(IndexState indexState, IndexName protoRequest) throws HandlerException {
    final ShardState shardState = indexState.getShard(0);
    if (shardState.isPrimary() == false) {
        throw new IllegalArgumentException("index \"" + shardState.name + "\" is either not started or is not a primary index");
    }
    SearcherVersion.Builder searchverVersionBuilder = SearcherVersion.newBuilder();
    try {
        if (shardState.nrtPrimaryNode.flushAndRefresh()) {
            // Something did get flushed (there were indexing ops since the last flush):
            // nocommit: we used to notify caller of the version, before trying to push to replicas, in
            // case we crash after flushing but
            // before notifying all replicas, at which point we have a newer version index than client
            // knew about?
            long version = shardState.nrtPrimaryNode.getCopyStateVersion();
            Queue<NRTPrimaryNode.ReplicaDetails> replicasInfos = shardState.nrtPrimaryNode.replicasInfos;
            shardState.nrtPrimaryNode.message("send flushed version=" + version + " replica count " + replicasInfos.size());
            // Notify current replicas:
            Iterator<NRTPrimaryNode.ReplicaDetails> it = replicasInfos.iterator();
            while (it.hasNext()) {
                NRTPrimaryNode.ReplicaDetails replicaDetails = it.next();
                int replicaID = replicaDetails.getReplicaId();
                ReplicationServerClient currentReplicaServerClient = replicaDetails.getReplicationServerClient();
                try {
                    // TODO: we should use multicast to broadcast files out to replicas
                    // TODO: ... replicas could copy from one another instead of just primary
                    // TODO: we could also prioritize one replica at a time?
                    currentReplicaServerClient.newNRTPoint(indexState.name, shardState.nrtPrimaryNode.getPrimaryGen(), version);
                } catch (StatusRuntimeException e) {
                    Status status = e.getStatus();
                    if (status.getCode().equals(Status.UNAVAILABLE.getCode())) {
                        logger.info("NRTPRimaryNode: sendNRTPoint, lost connection to replicaId: " + replicaDetails.getReplicaId() + " host: " + replicaDetails.getHostPort().getHostName() + " port: " + replicaDetails.getHostPort().getPort());
                        it.remove();
                    }
                } catch (Exception e) {
                    shardState.nrtPrimaryNode.message("top: failed to connect R" + replicaID + " for newNRTPoint; skipping: " + e.getMessage());
                    logger.info("top: failed to connect R" + replicaID + " for newNRTPoint; skipping: " + e.getMessage());
                }
            }
            return searchverVersionBuilder.setVersion(version).setDidRefresh(true).build();
        } else {
            SearcherTaxonomyManager.SearcherAndTaxonomy s = shardState.acquire();
            try {
                long version = ((DirectoryReader) s.searcher.getIndexReader()).getVersion();
                return searchverVersionBuilder.setVersion(version).setDidRefresh(false).build();
            } finally {
                shardState.release(s);
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}