Examples with BulkRequestBuilder org.elasticsearch.action.bulk.BulkRequestBuilder used on opensource projects

Example 1 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class SearchTemplateIT method testIndexedTemplateWithArray.

public void testIndexedTemplateWithArray() throws Exception {
    String multiQuery = "{\"query\":{\"terms\":{\"theField\":[\"{{#fieldParam}}\",\"{{.}}\",\"{{/fieldParam}}\"]}}}";
    assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("4").setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON));
    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
    bulkRequestBuilder.get();
    client().admin().indices().prepareRefresh().get();
    Map<String, Object> arrayTemplateParams = new HashMap<>();
    String[] fieldParams = { "foo", "bar" };
    arrayTemplateParams.put("fieldParam", fieldParams);
    SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("4").setScriptType(ScriptType.STORED).setScriptParams(arrayTemplateParams).get();
    assertHitCount(searchResponse.getResponse(), 5);
}

Example 2 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class SearchTemplateIT method testIndexedTemplateClient.

public void testIndexedTemplateClient() throws Exception {
    assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("testTemplate").setContent(new BytesArray("{" + "\"template\":{" + "                \"query\":{" + "                   \"match\":{" + "                    \"theField\" : \"{{fieldParam}}\"}" + "       }" + "}" + "}"), XContentType.JSON));
    assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("testTemplate").setContent(new BytesArray("{" + "\"template\":{" + "                \"query\":{" + "                   \"match\":{" + "                    \"theField\" : \"{{fieldParam}}\"}" + "       }" + "}" + "}"), XContentType.JSON));
    GetStoredScriptResponse getResponse = client().admin().cluster().prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
    assertNotNull(getResponse.getSource());
    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
    bulkRequestBuilder.get();
    client().admin().indices().prepareRefresh().get();
    Map<String, Object> templateParams = new HashMap<>();
    templateParams.put("fieldParam", "foo");
    SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("testTemplate").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get();
    assertHitCount(searchResponse.getResponse(), 4);
    assertAcked(client().admin().cluster().prepareDeleteStoredScript(MustacheScriptEngineService.NAME, "testTemplate"));
    getResponse = client().admin().cluster().prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
    assertNull(getResponse.getSource());
    IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("/template_index/mustache/1000").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get());
    assertThat(e.getMessage(), containsString("illegal stored script format [/template_index/mustache/1000] use only <id>"));
}

Example 3 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class SearchTemplateIT method testIndexedTemplate.

public void testIndexedTemplate() throws Exception {
    assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("1a").setContent(new BytesArray("{" + "\"template\":{" + "                \"query\":{" + "                   \"match\":{" + "                    \"theField\" : \"{{fieldParam}}\"}" + "       }" + "}" + "}"), XContentType.JSON));
    assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("2").setContent(new BytesArray("{" + "\"template\":{" + "                \"query\":{" + "                   \"match\":{" + "                    \"theField\" : \"{{fieldParam}}\"}" + "       }" + "}" + "}"), XContentType.JSON));
    assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("3").setContent(new BytesArray("{" + "\"template\":{" + "             \"match\":{" + "                    \"theField\" : \"{{fieldParam}}\"}" + "       }" + "}"), XContentType.JSON));
    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
    bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
    bulkRequestBuilder.get();
    client().admin().indices().prepareRefresh().get();
    Map<String, Object> templateParams = new HashMap<>();
    templateParams.put("fieldParam", "foo");
    SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest().indices("test").types("type")).setScript("1a").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get();
    assertHitCount(searchResponse.getResponse(), 4);
    expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest().indices("test").types("type")).setScript("/template_index/mustache/1000").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get());
    expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest().indices("test").types("type")).setScript("/myindex/mustache/1").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get());
    templateParams.put("fieldParam", "bar");
    searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("/mustache/2").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get();
    assertHitCount(searchResponse.getResponse(), 1);
    assertWarnings("use of </lang/id> [/mustache/2] for looking up" + " stored scripts/templates has been deprecated, use only <id> [2] instead");
    Map<String, Object> vars = new HashMap<>();
    vars.put("fieldParam", "bar");
    TemplateQueryBuilder builder = new TemplateQueryBuilder("3", ScriptType.STORED, vars);
    SearchResponse sr = client().prepareSearch().setQuery(builder).execute().actionGet();
    assertHitCount(sr, 1);
    assertWarnings("[template] query is deprecated, use search template api instead");
}

Example 4 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class ESIntegTestCase method indexRandom.

/**
     * Indexes the given {@link IndexRequestBuilder} instances randomly. It shuffles the given builders and either
     * indexes them in a blocking or async fashion. This is very useful to catch problems that relate to internal document
     * ids or index segment creations. Some features might have bug when a given document is the first or the last in a
     * segment or if only one document is in a segment etc. This method prevents issues like this by randomizing the index
     * layout.
     *
     * @param forceRefresh   if <tt>true</tt> all involved indices are refreshed once the documents are indexed.
     * @param dummyDocuments if <tt>true</tt> some empty dummy documents may be randomly inserted into the document list and deleted once
     *                       all documents are indexed. This is useful to produce deleted documents on the server side.
     * @param maybeFlush     if <tt>true</tt> this method may randomly execute full flushes after index operations.
     * @param builders       the documents to index.
     */
public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders) throws InterruptedException, ExecutionException {
    Random random = random();
    Set<String> indicesSet = new HashSet<>();
    for (IndexRequestBuilder builder : builders) {
        indicesSet.add(builder.request().index());
    }
    Set<Tuple<String, String>> bogusIds = new HashSet<>();
    if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) {
        builders = new ArrayList<>(builders);
        final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
        // inject some bogus docs
        final int numBogusDocs = scaledRandomIntBetween(1, builders.size() * 2);
        final int unicodeLen = between(1, 10);
        for (int i = 0; i < numBogusDocs; i++) {
            String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
            String index = RandomPicks.randomFrom(random, indices);
            bogusIds.add(new Tuple<>(index, id));
            builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON));
        }
    }
    final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
    Collections.shuffle(builders, random());
    final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
    List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
    // If you are indexing just a few documents then frequently do it one at a time.  If many then frequently in bulk.
    if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) {
        if (frequently()) {
            logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false);
            for (IndexRequestBuilder indexRequestBuilder : builders) {
                indexRequestBuilder.execute(new PayloadLatchedActionListener<IndexResponse, IndexRequestBuilder>(indexRequestBuilder, newLatch(inFlightAsyncOperations), errors));
                postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
            }
        } else {
            logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), false, false);
            for (IndexRequestBuilder indexRequestBuilder : builders) {
                indexRequestBuilder.execute().actionGet();
                postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
            }
        }
    } else {
        List<List<IndexRequestBuilder>> partition = eagerPartition(builders, Math.min(MAX_BULK_INDEX_REQUEST_SIZE, Math.max(1, (int) (builders.size() * randomDouble()))));
        logger.info("Index [{}] docs async: [{}] bulk: [{}] partitions [{}]", builders.size(), false, true, partition.size());
        for (List<IndexRequestBuilder> segmented : partition) {
            BulkRequestBuilder bulkBuilder = client().prepareBulk();
            for (IndexRequestBuilder indexRequestBuilder : segmented) {
                bulkBuilder.add(indexRequestBuilder);
            }
            BulkResponse actionGet = bulkBuilder.execute().actionGet();
            assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false));
        }
    }
    for (CountDownLatch operation : inFlightAsyncOperations) {
        operation.await();
    }
    final List<Exception> actualErrors = new ArrayList<>();
    for (Tuple<IndexRequestBuilder, Exception> tuple : errors) {
        if (ExceptionsHelper.unwrapCause(tuple.v2()) instanceof EsRejectedExecutionException) {
            // re-index if rejected
            tuple.v1().execute().actionGet();
        } else {
            actualErrors.add(tuple.v2());
        }
    }
    assertThat(actualErrors, emptyIterable());
    if (!bogusIds.isEmpty()) {
        // delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
        for (Tuple<String, String> doc : bogusIds) {
            assertEquals("failed to delete a dummy doc [" + doc.v1() + "][" + doc.v2() + "]", DocWriteResponse.Result.DELETED, client().prepareDelete(doc.v1(), RANDOM_BOGUS_TYPE, doc.v2()).get().getResult());
        }
    }
    if (forceRefresh) {
        assertNoFailures(client().admin().indices().prepareRefresh(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
    }
}

Example 5 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch-river-kafka by endgameinc.

the class JsonMessageHandlerTest method testIt.

public void testIt() throws Exception {
    Client client = createMock(Client.class);
    IndexRequestBuilder irb = createMock(IndexRequestBuilder.class);
    JsonMessageHandler h = new JsonMessageHandler(client);
    byte[] json = toJson(rec).getBytes();
    expect(client.prepareIndex(anyObject(String.class), anyObject(String.class), anyObject(String.class))).andReturn(irb);
    replay(client);
    Message message = createMock(Message.class);
    expect(message.payload()).andReturn(ByteBuffer.wrap(json));
    replay(message);
    BulkRequestBuilder bulkRequestBuilder = createMock(BulkRequestBuilder.class);
    expect(bulkRequestBuilder.add(anyObject(IndexRequestBuilder.class))).andReturn(null);
    replay(bulkRequestBuilder);
    try {
        h.handle(bulkRequestBuilder, message);
    } catch (Exception e) {
        fail("This should not fail");
    }
    verify(client);
}