Examples with AddDocumentRequest com.yelp.nrtsearch.server.grpc.AddDocumentRequest used on opensource projects

Example 11 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class BucketedTieredMergePolicyTest method addData.

private void addData(int count) throws Exception {
    IndexWriter writer = getGlobalState().getIndex(DEFAULT_TEST_INDEX).getShard(0).writer;
    for (int i = 0; i < count; ++i) {
        AddDocumentRequest request = AddDocumentRequest.newBuilder().setIndexName(DEFAULT_TEST_INDEX).putFields("doc_id", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(i)).build()).putFields("int_score", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(i + 1)).build()).putFields("int_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(i + 2)).build()).build();
        addDocuments(Stream.of(request));
        writer.flush();
    }
    getGlobalState().getIndex(DEFAULT_TEST_INDEX).getShard(0).maybeRefreshBlocking();
}

Example 12 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class TestIndexManager method setUpIndex.

protected static void setUpIndex(LuceneServerClient standaloneServerClient, String indexName, String suggestionsFilePath, OneDocBuilder oneDocBuilder) throws IOException, ExecutionException, InterruptedException {
    // create index if it does not exist
    try {
        createIndex(standaloneServerClient, indexName);
    } catch (StatusRuntimeException e) {
        if (!e.getStatus().getCode().name().equals("ALREADY_EXISTS"))
            throw e;
    }
    // add live settings
    liveSettings(standaloneServerClient, indexName);
    // register fields
    registerFields(standaloneServerClient, Paths.get("src", "test", "resources", "registerFieldsYelpSuggestTestPayload.json").toAbsolutePath().toString());
    // start index
    StartIndexRequest startIndexRequest = StartIndexRequest.newBuilder().setIndexName(indexName).setMode(Mode.STANDALONE).setPrimaryGen(0).build();
    startIndex(standaloneServerClient, startIndexRequest);
    // index docs
    long t1 = System.nanoTime();
    Stream.Builder<AddDocumentRequest> builder = Stream.builder();
    final ExecutorService indexService = YelpReviewsTest.createExecutorService((Runtime.getRuntime().availableProcessors()) / 4, "LuceneIndexing");
    Path suggestionsPath = Paths.get(suggestionsFilePath);
    List<Future<Long>> results = ParallelDocumentIndexer.buildAndIndexDocs(oneDocBuilder, suggestionsPath, indexService, standaloneServerClient);
    // wait till all indexing done
    for (Future<Long> each : results) {
        try {
            Long genId = each.get();
            logger.info("ParallelDocumentIndexer.buildAndIndexDocs returned genId: {}", genId);
        } catch (ExecutionException | InterruptedException futureException) {
            System.out.println(futureException.getCause());
        }
    }
    long t2 = System.nanoTime();
    System.out.println(String.format("IT took %s nanosecs to index documents", (t2 - t1)));
    // commit
    standaloneServerClient.getBlockingStub().commit(CommitRequest.newBuilder().setIndexName(indexName).build());
}

Example 13 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class IndexerTask method index.

public Long index(LuceneServerClient luceneServerClient, Stream<AddDocumentRequest> addDocumentRequestStream) throws Exception {
    String threadId = Thread.currentThread().getName() + Thread.currentThread().getId();
    final CountDownLatch finishLatch = new CountDownLatch(1);
    StreamObserver<AddDocumentResponse> responseObserver = new StreamObserver<>() {

        @Override
        public void onNext(AddDocumentResponse value) {
            // Note that Server sends back only 1 message (Unary mode i.e. Server calls its onNext
            // only once
            // which is when it is done with indexing the entire stream), which means this method
            // should be
            // called only once.
            logger.debug("Received response for genId: {} on threadId: {}", value.getGenId(), threadId);
            genId = value.getGenId();
        }

        @Override
        public void onError(Throwable t) {
            logger.error(t.getMessage(), t);
            finishLatch.countDown();
        }

        @Override
        public void onCompleted() {
            logger.debug("Received final response from server on threadId: {}", threadId);
            finishLatch.countDown();
        }
    };
    // The responseObserver handles responses from the server (i.e. 1 onNext and 1 completed)
    // The requestObserver handles the sending of stream of client requests to server (i.e. multiple
    // onNext and 1 completed)
    StreamObserver<AddDocumentRequest> requestObserver = luceneServerClient.getAsyncStub().addDocuments(responseObserver);
    try {
        addDocumentRequestStream.forEach(addDocumentRequest -> requestObserver.onNext(addDocumentRequest));
    } catch (RuntimeException e) {
        // Cancel RPC
        requestObserver.onError(e);
        throw e;
    }
    // Mark the end of requests
    requestObserver.onCompleted();
    logger.debug("sent async addDocumentsRequest to server on threadId: {}", threadId);
    // Receiving happens asynchronously, so block here for 5 minutes
    if (!finishLatch.await(5, TimeUnit.MINUTES)) {
        logger.warn("addDocuments can not finish within 5 minutes on threadId: {}", threadId);
    }
    return Long.valueOf(genId);
}

Example 14 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class NodeNameResolverAndLoadBalancingTests method startIndexAndAddDocuments.

private void startIndexAndAddDocuments(GrpcServer server, int id) throws InterruptedException, IOException {
    LuceneServerGrpc.LuceneServerBlockingStub stub = server.getBlockingStub();
    stub.createIndex(CreateIndexRequest.newBuilder().setIndexName(TEST_INDEX).build());
    FieldDefRequest fieldDefRequest = FieldDefRequest.newBuilder().setIndexName(TEST_INDEX).addField(Field.newBuilder().setName(FIELD_NAME).setType(FieldType.INT).setSearch(true).setStoreDocValues(true).build()).build();
    stub.registerFields(fieldDefRequest);
    stub.startIndex(StartIndexRequest.newBuilder().setIndexName(TEST_INDEX).build());
    AddDocumentRequest addDocumentRequest = AddDocumentRequest.newBuilder().setIndexName(TEST_INDEX).putFields(FIELD_NAME, AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).build();
    new GrpcServer.TestServer(server, false, Mode.STANDALONE).addDocumentsFromStream(Stream.of(addDocumentRequest));
    stub.commit(CommitRequest.newBuilder().setIndexName(TEST_INDEX).build());
    stub.refresh(RefreshRequest.newBuilder().setIndexName(TEST_INDEX).build());
}

Example 15 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class CollectorStatsWrapperTest method initIndex.

@Override
public void initIndex(String name) throws Exception {
    IndexWriter writer = getGlobalState().getIndex(name).getShard(0).writer;
    // don't want any merges for these tests
    writer.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
    // create a shuffled list of ids
    List<Integer> idList = new ArrayList<>();
    for (int i = 0; i < NUM_DOCS; ++i) {
        idList.add(i);
    }
    Collections.shuffle(idList);
    // add documents one chunk at a time to ensure multiple index segments
    List<AddDocumentRequest> requestChunk = new ArrayList<>();
    for (Integer id : idList) {
        requestChunk.add(AddDocumentRequest.newBuilder().setIndexName(name).putFields("doc_id", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).putFields("int_score", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(NUM_DOCS - id)).build()).putFields("int_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).build());
        if (requestChunk.size() == SEGMENT_CHUNK) {
            addDocuments(requestChunk.stream());
            requestChunk.clear();
            writer.commit();
        }
    }
}