Examples with AddDocumentRequest com.yelp.nrtsearch.server.grpc.AddDocumentRequest used on opensource projects

Example 21 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class EagerGlobalOrdinalsTest method addDocAndRefresh.

private void addDocAndRefresh() throws Exception {
    AddDocumentRequest request = AddDocumentRequest.newBuilder().setIndexName(DEFAULT_TEST_INDEX).putFields(NOT_EAGER_FIELD, MultiValuedField.newBuilder().addValue("v1").addValue("v2").build()).putFields(EAGER_FIELD, MultiValuedField.newBuilder().addValue("v3").build()).build();
    addDocuments(Stream.of(request));
    getGrpcServer().getBlockingStub().refresh(RefreshRequest.newBuilder().setIndexName(DEFAULT_TEST_INDEX).build());
}

Example 22 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class FacetScriptFacetsTest method initIndex.

protected void initIndex(String name) throws Exception {
    IndexWriter writer = getGlobalState().getIndex(name).getShard(0).writer;
    // don't want any merges for these tests
    writer.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
    // add documents one chunk at a time to ensure multiple index segments
    List<AddDocumentRequest> requestChunk = new ArrayList<>();
    for (int id = 0; id < NUM_DOCS; ++id) {
        requestChunk.add(AddDocumentRequest.newBuilder().setIndexName(name).putFields("doc_id", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).putFields("int_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).putFields("atom_1", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id % 3)).build()).putFields("atom_2", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id % 2)).build()).build());
        if (requestChunk.size() == SEGMENT_CHUNK) {
            addDocuments(requestChunk.stream());
            requestChunk.clear();
            writer.commit();
        }
    }
}

Example 23 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class FacetTopHitsTest method initIndex.

protected void initIndex(String name) throws Exception {
    IndexWriter writer = getGlobalState().getIndex(name).getShard(0).writer;
    // don't want any merges for these tests
    writer.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
    // add documents one chunk at a time to ensure multiple index segments
    List<AddDocumentRequest> requestChunk = new ArrayList<>();
    for (int id = 0; id < NUM_DOCS; ++id) {
        requestChunk.add(AddDocumentRequest.newBuilder().setIndexName(name).putFields("doc_id", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).putFields("int_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf((id + 25) % NUM_DOCS)).build()).putFields("long_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id / 10)).build()).build());
        if (requestChunk.size() == SEGMENT_CHUNK) {
            addDocuments(requestChunk.stream());
            requestChunk.clear();
            writer.commit();
        }
    }
}

Example 24 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class AddDocumentsCommand method call.

@Override
public Integer call() throws Exception {
    LuceneServerClient client = baseCmd.getClient();
    try {
        String indexName = getIndexName();
        String fileType = getFileType();
        Stream<AddDocumentRequest> addDocumentRequestStream;
        Path filePath = Paths.get(getFileName());
        if (fileType.equalsIgnoreCase("csv")) {
            Reader reader = Files.newBufferedReader(filePath);
            CSVParser csvParser = new CSVParser(reader, CSVFormat.DEFAULT.withFirstRecordAsHeader());
            addDocumentRequestStream = new LuceneServerClientBuilder.AddDocumentsClientBuilder(indexName, csvParser).buildRequest(filePath);
            client.addDocuments(addDocumentRequestStream);
        } else if (fileType.equalsIgnoreCase("json")) {
            LuceneServerClientBuilder.AddJsonDocumentsClientBuilder addJsonDocumentsClientBuilder = new LuceneServerClientBuilder.AddJsonDocumentsClientBuilder(indexName, new Gson(), filePath, getMaxBufferLen());
            while (!addJsonDocumentsClientBuilder.isFinished()) {
                addDocumentRequestStream = addJsonDocumentsClientBuilder.buildRequest(filePath);
                client.addDocuments(addDocumentRequestStream);
            }
        } else {
            throw new RuntimeException(String.format("%s is not a valid fileType", fileType));
        }
    } finally {
        client.shutdown();
    }
    return 0;
}

Example 25 with AddDocumentRequest

use of com.yelp.nrtsearch.server.grpc.AddDocumentRequest in project nrtsearch by Yelp.

the class SearchStatsWrapperTest method initIndex.

@Override
public void initIndex(String name) throws Exception {
    IndexWriter writer = getGlobalState().getIndex(name).getShard(0).writer;
    // don't want any merges for these tests
    writer.getConfig().setMergePolicy(NoMergePolicy.INSTANCE);
    // create a shuffled list of ids
    List<Integer> idList = new ArrayList<>();
    for (int i = 0; i < NUM_DOCS; ++i) {
        idList.add(i);
    }
    Collections.shuffle(idList);
    // add documents one chunk at a time to ensure multiple index segments
    List<AddDocumentRequest> requestChunk = new ArrayList<>();
    for (Integer id : idList) {
        requestChunk.add(AddDocumentRequest.newBuilder().setIndexName(name).putFields("doc_id", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).putFields("int_score", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(NUM_DOCS - id)).build()).putFields("int_field", AddDocumentRequest.MultiValuedField.newBuilder().addValue(String.valueOf(id)).build()).build());
        if (requestChunk.size() == SEGMENT_CHUNK) {
            addDocuments(requestChunk.stream());
            requestChunk.clear();
            writer.commit();
        }
    }
}