use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.
the class SearchTemplateIT method testIndexedTemplateWithArray.
public void testIndexedTemplateWithArray() throws Exception {
String multiQuery = "{\"query\":{\"terms\":{\"theField\":[\"{{#fieldParam}}\",\"{{.}}\",\"{{/fieldParam}}\"]}}}";
assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("4").setContent(jsonBuilder().startObject().field("template", multiQuery).endObject().bytes(), XContentType.JSON));
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
bulkRequestBuilder.get();
client().admin().indices().prepareRefresh().get();
Map<String, Object> arrayTemplateParams = new HashMap<>();
String[] fieldParams = { "foo", "bar" };
arrayTemplateParams.put("fieldParam", fieldParams);
SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("4").setScriptType(ScriptType.STORED).setScriptParams(arrayTemplateParams).get();
assertHitCount(searchResponse.getResponse(), 5);
}
use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.
the class SearchTemplateIT method testIndexedTemplateClient.
public void testIndexedTemplateClient() throws Exception {
assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("testTemplate").setContent(new BytesArray("{" + "\"template\":{" + " \"query\":{" + " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + "}"), XContentType.JSON));
assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("testTemplate").setContent(new BytesArray("{" + "\"template\":{" + " \"query\":{" + " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + "}"), XContentType.JSON));
GetStoredScriptResponse getResponse = client().admin().cluster().prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
assertNotNull(getResponse.getSource());
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
bulkRequestBuilder.get();
client().admin().indices().prepareRefresh().get();
Map<String, Object> templateParams = new HashMap<>();
templateParams.put("fieldParam", "foo");
SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("testTemplate").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get();
assertHitCount(searchResponse.getResponse(), 4);
assertAcked(client().admin().cluster().prepareDeleteStoredScript(MustacheScriptEngineService.NAME, "testTemplate"));
getResponse = client().admin().cluster().prepareGetStoredScript(MustacheScriptEngineService.NAME, "testTemplate").get();
assertNull(getResponse.getSource());
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("/template_index/mustache/1000").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get());
assertThat(e.getMessage(), containsString("illegal stored script format [/template_index/mustache/1000] use only <id>"));
}
use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.
the class SearchTemplateIT method testIndexedTemplate.
public void testIndexedTemplate() throws Exception {
assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("1a").setContent(new BytesArray("{" + "\"template\":{" + " \"query\":{" + " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + "}"), XContentType.JSON));
assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("2").setContent(new BytesArray("{" + "\"template\":{" + " \"query\":{" + " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}" + "}"), XContentType.JSON));
assertAcked(client().admin().cluster().preparePutStoredScript().setLang(MustacheScriptEngineService.NAME).setId("3").setContent(new BytesArray("{" + "\"template\":{" + " \"match\":{" + " \"theField\" : \"{{fieldParam}}\"}" + " }" + "}"), XContentType.JSON));
BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
bulkRequestBuilder.add(client().prepareIndex("test", "type", "1").setSource("{\"theField\":\"foo\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "2").setSource("{\"theField\":\"foo 2\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "3").setSource("{\"theField\":\"foo 3\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "4").setSource("{\"theField\":\"foo 4\"}", XContentType.JSON));
bulkRequestBuilder.add(client().prepareIndex("test", "type", "5").setSource("{\"theField\":\"bar\"}", XContentType.JSON));
bulkRequestBuilder.get();
client().admin().indices().prepareRefresh().get();
Map<String, Object> templateParams = new HashMap<>();
templateParams.put("fieldParam", "foo");
SearchTemplateResponse searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest().indices("test").types("type")).setScript("1a").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get();
assertHitCount(searchResponse.getResponse(), 4);
expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest().indices("test").types("type")).setScript("/template_index/mustache/1000").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get());
expectThrows(IllegalArgumentException.class, () -> new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest().indices("test").types("type")).setScript("/myindex/mustache/1").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get());
templateParams.put("fieldParam", "bar");
searchResponse = new SearchTemplateRequestBuilder(client()).setRequest(new SearchRequest("test").types("type")).setScript("/mustache/2").setScriptType(ScriptType.STORED).setScriptParams(templateParams).get();
assertHitCount(searchResponse.getResponse(), 1);
assertWarnings("use of </lang/id> [/mustache/2] for looking up" + " stored scripts/templates has been deprecated, use only <id> [2] instead");
Map<String, Object> vars = new HashMap<>();
vars.put("fieldParam", "bar");
TemplateQueryBuilder builder = new TemplateQueryBuilder("3", ScriptType.STORED, vars);
SearchResponse sr = client().prepareSearch().setQuery(builder).execute().actionGet();
assertHitCount(sr, 1);
assertWarnings("[template] query is deprecated, use search template api instead");
}
use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.
the class ESIntegTestCase method indexRandom.
/**
* Indexes the given {@link IndexRequestBuilder} instances randomly. It shuffles the given builders and either
* indexes them in a blocking or async fashion. This is very useful to catch problems that relate to internal document
* ids or index segment creations. Some features might have bug when a given document is the first or the last in a
* segment or if only one document is in a segment etc. This method prevents issues like this by randomizing the index
* layout.
*
* @param forceRefresh if <tt>true</tt> all involved indices are refreshed once the documents are indexed.
* @param dummyDocuments if <tt>true</tt> some empty dummy documents may be randomly inserted into the document list and deleted once
* all documents are indexed. This is useful to produce deleted documents on the server side.
* @param maybeFlush if <tt>true</tt> this method may randomly execute full flushes after index operations.
* @param builders the documents to index.
*/
public void indexRandom(boolean forceRefresh, boolean dummyDocuments, boolean maybeFlush, List<IndexRequestBuilder> builders) throws InterruptedException, ExecutionException {
Random random = random();
Set<String> indicesSet = new HashSet<>();
for (IndexRequestBuilder builder : builders) {
indicesSet.add(builder.request().index());
}
Set<Tuple<String, String>> bogusIds = new HashSet<>();
if (random.nextBoolean() && !builders.isEmpty() && dummyDocuments) {
builders = new ArrayList<>(builders);
final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
// inject some bogus docs
final int numBogusDocs = scaledRandomIntBetween(1, builders.size() * 2);
final int unicodeLen = between(1, 10);
for (int i = 0; i < numBogusDocs; i++) {
String id = randomRealisticUnicodeOfLength(unicodeLen) + Integer.toString(dummmyDocIdGenerator.incrementAndGet());
String index = RandomPicks.randomFrom(random, indices);
bogusIds.add(new Tuple<>(index, id));
builders.add(client().prepareIndex(index, RANDOM_BOGUS_TYPE, id).setSource("{}", XContentType.JSON));
}
}
final String[] indices = indicesSet.toArray(new String[indicesSet.size()]);
Collections.shuffle(builders, random());
final CopyOnWriteArrayList<Tuple<IndexRequestBuilder, Exception>> errors = new CopyOnWriteArrayList<>();
List<CountDownLatch> inFlightAsyncOperations = new ArrayList<>();
// If you are indexing just a few documents then frequently do it one at a time. If many then frequently in bulk.
if (builders.size() < FREQUENT_BULK_THRESHOLD ? frequently() : builders.size() < ALWAYS_BULK_THRESHOLD ? rarely() : false) {
if (frequently()) {
logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), true, false);
for (IndexRequestBuilder indexRequestBuilder : builders) {
indexRequestBuilder.execute(new PayloadLatchedActionListener<IndexResponse, IndexRequestBuilder>(indexRequestBuilder, newLatch(inFlightAsyncOperations), errors));
postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
}
} else {
logger.info("Index [{}] docs async: [{}] bulk: [{}]", builders.size(), false, false);
for (IndexRequestBuilder indexRequestBuilder : builders) {
indexRequestBuilder.execute().actionGet();
postIndexAsyncActions(indices, inFlightAsyncOperations, maybeFlush);
}
}
} else {
List<List<IndexRequestBuilder>> partition = eagerPartition(builders, Math.min(MAX_BULK_INDEX_REQUEST_SIZE, Math.max(1, (int) (builders.size() * randomDouble()))));
logger.info("Index [{}] docs async: [{}] bulk: [{}] partitions [{}]", builders.size(), false, true, partition.size());
for (List<IndexRequestBuilder> segmented : partition) {
BulkRequestBuilder bulkBuilder = client().prepareBulk();
for (IndexRequestBuilder indexRequestBuilder : segmented) {
bulkBuilder.add(indexRequestBuilder);
}
BulkResponse actionGet = bulkBuilder.execute().actionGet();
assertThat(actionGet.hasFailures() ? actionGet.buildFailureMessage() : "", actionGet.hasFailures(), equalTo(false));
}
}
for (CountDownLatch operation : inFlightAsyncOperations) {
operation.await();
}
final List<Exception> actualErrors = new ArrayList<>();
for (Tuple<IndexRequestBuilder, Exception> tuple : errors) {
if (ExceptionsHelper.unwrapCause(tuple.v2()) instanceof EsRejectedExecutionException) {
// re-index if rejected
tuple.v1().execute().actionGet();
} else {
actualErrors.add(tuple.v2());
}
}
assertThat(actualErrors, emptyIterable());
if (!bogusIds.isEmpty()) {
// delete the bogus types again - it might trigger merges or at least holes in the segments and enforces deleted docs!
for (Tuple<String, String> doc : bogusIds) {
assertEquals("failed to delete a dummy doc [" + doc.v1() + "][" + doc.v2() + "]", DocWriteResponse.Result.DELETED, client().prepareDelete(doc.v1(), RANDOM_BOGUS_TYPE, doc.v2()).get().getResult());
}
}
if (forceRefresh) {
assertNoFailures(client().admin().indices().prepareRefresh(indices).setIndicesOptions(IndicesOptions.lenientExpandOpen()).execute().get());
}
}
use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch-river-kafka by endgameinc.
the class JsonMessageHandlerTest method testIt.
public void testIt() throws Exception {
Client client = createMock(Client.class);
IndexRequestBuilder irb = createMock(IndexRequestBuilder.class);
JsonMessageHandler h = new JsonMessageHandler(client);
byte[] json = toJson(rec).getBytes();
expect(client.prepareIndex(anyObject(String.class), anyObject(String.class), anyObject(String.class))).andReturn(irb);
replay(client);
Message message = createMock(Message.class);
expect(message.payload()).andReturn(ByteBuffer.wrap(json));
replay(message);
BulkRequestBuilder bulkRequestBuilder = createMock(BulkRequestBuilder.class);
expect(bulkRequestBuilder.add(anyObject(IndexRequestBuilder.class))).andReturn(null);
replay(bulkRequestBuilder);
try {
h.handle(bulkRequestBuilder, message);
} catch (Exception e) {
fail("This should not fail");
}
verify(client);
}
Aggregations