Examples with AddDocumentRequest com.yelp.nrtsearch.server.grpc.AddDocumentRequest used on opensource projects

Example 1 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class ServerTestCase method addDocsFromResourceFile.

public static void addDocsFromResourceFile(String index, String resourceFile) throws Exception {
    Path filePath = Paths.get(ServerTestCase.class.getResource(resourceFile).toURI());
    Reader reader = Files.newBufferedReader(filePath);
    CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT.withFirstRecordAsHeader());
    Stream<AddDocumentRequest> requestStream = new LuceneServerClientBuilder.AddDocumentsClientBuilder(index, csvParser).buildRequest(filePath);
    addDocuments(requestStream);
}

Example 2 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class ServerTestCase method addDocuments.

public static AddDocumentResponse addDocuments(Stream<AddDocumentRequest> requestStream) throws Exception {
    CountDownLatch finishLatch = new CountDownLatch(1);
    // observers responses from Server(should get one onNext and oneCompleted)
    final AtomicReference<AddDocumentResponse> response = new AtomicReference<>();
    final AtomicReference<Exception> exception = new AtomicReference<>();
    StreamObserver<AddDocumentResponse> responseStreamObserver = new StreamObserver<>() {

        @Override
        public void onNext(AddDocumentResponse value) {
            response.set(value);
        }

        @Override
        public void onError(Throwable t) {
            exception.set(new RuntimeException(t));
            finishLatch.countDown();
        }

        @Override
        public void onCompleted() {
            finishLatch.countDown();
        }
    };
    // requestObserver sends requests to Server (one onNext per AddDocumentRequest and one
    // onCompleted)
    StreamObserver<AddDocumentRequest> requestObserver = grpcServer.getStub().addDocuments(responseStreamObserver);
    // parse CSV into a stream of AddDocumentRequest
    try {
        requestStream.forEach(requestObserver::onNext);
    } catch (RuntimeException e) {
        // Cancel RPC
        requestObserver.onError(e);
        throw e;
    }
    // Mark the end of requests
    requestObserver.onCompleted();
    // Receiving happens asynchronously, so block here 20 seconds
    if (!finishLatch.await(20, TimeUnit.SECONDS)) {
        throw new RuntimeException("addDocuments can not finish within 20 seconds");
    }
    // Re-throw exception
    if (exception.get() != null) {
        throw exception.get();
    }
    return response.get();
}

Example 3 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class EagerGlobalOrdinalsTest method initIndex.

protected void initIndex(String name) throws Exception {
    List<AddDocumentRequest> docs = new ArrayList<>();
    AddDocumentRequest request = AddDocumentRequest.newBuilder().setIndexName(name).putFields(NOT_EAGER_FIELD, MultiValuedField.newBuilder().addValue("v1").addValue("v2").build()).putFields(EAGER_FIELD, MultiValuedField.newBuilder().addValue("v3").build()).build();
    docs.add(request);
    request = AddDocumentRequest.newBuilder().setIndexName(name).putFields(NOT_EAGER_FIELD, MultiValuedField.newBuilder().addValue("v1").build()).putFields(EAGER_FIELD, MultiValuedField.newBuilder().addValue("v3").addValue("v4").build()).build();
    docs.add(request);
    addDocuments(docs.stream());
}

Example 4 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class FilteredSSDVFacetCountsTest method initIndex.

@Override
public void initIndex(String name) throws Exception {
    IndexWriter writer = getGlobalState().getIndex(name).getShard(0).writer;
    // don't want any merges for these tests
    writer.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
    AddDocumentRequest doc1 = AddDocumentRequest.newBuilder().setIndexName(name).putFields("sorted_doc_values_facet_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue("M1").addValue("M2").build()).putFields("sorted_doc_values_facet_field_single_valued", AddDocumentRequest.MultiValuedField.newBuilder().addValue("S1").build()).build();
    AddDocumentRequest doc2 = AddDocumentRequest.newBuilder().setIndexName(name).putFields("sorted_doc_values_facet_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue("M2").build()).putFields("sorted_doc_values_facet_field_single_valued", AddDocumentRequest.MultiValuedField.newBuilder().addValue("S1").build()).build();
    AddDocumentRequest doc3 = AddDocumentRequest.newBuilder().setIndexName(name).putFields("sorted_doc_values_facet_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue("M1").addValue("M2").addValue("M3").build()).putFields("sorted_doc_values_facet_field_single_valued", AddDocumentRequest.MultiValuedField.newBuilder().addValue("S2").build()).build();
    if (MULTI_SEGMENT_INDEX.equals(name)) {
        addDocuments(Stream.of(doc1));
        writer.commit();
        addDocuments(Stream.of(doc2));
        writer.commit();
        addDocuments(Stream.of(doc3));
        writer.commit();
    } else {
        List<AddDocumentRequest> documents = new ArrayList<>();
        documents.add(doc1);
        documents.add(doc2);
        documents.add(doc3);
        addDocuments(documents.stream());
    }
}

Example 5 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class TermsCollectorManagerTest method initIndex.

protected void initIndex(String name) throws Exception {
    IndexWriter writer = getGlobalState().getIndex(name).getShard(0).writer;
    // don't want any merges for these tests
    writer.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
    // add documents one chunk at a time to ensure multiple index segments
    List<AddDocumentRequest> requestChunk = new ArrayList<>();
    for (int id = 0; id < NUM_DOCS; ++id) {
        requestChunk.add(AddDocumentRequest.newBuilder().setIndexName(name).putFields("doc_id", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).putFields("int_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).putFields("atom_1", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id % 3)).build()).putFields("atom_2", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id % 2)).build()).build());
        if (requestChunk.size() == SEGMENT_CHUNK) {
            addDocuments(requestChunk.stream());
            requestChunk.clear();
            writer.commit();
        }
    }
}