Examples with IndexRequestBuilder org.elasticsearch.action.index.IndexRequestBuilder used on opensource projects

Example 31 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testSnapshotMoreThanOnce.

public void testSnapshotMoreThanOnce() throws ExecutionException, InterruptedException {
    Client client = client();
    logger.info("-->  creating repository");
    assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    // only one shard
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)));
    ensureGreen();
    logger.info("-->  indexing");
    final int numdocs = randomIntBetween(10, 100);
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test", "doc", Integer.toString(i)).setSource("foo", "bar" + i);
    }
    indexRandom(true, builders);
    flushAndRefresh();
    assertNoFailures(client().admin().indices().prepareForceMerge("test").setFlush(true).setMaxNumSegments(1).get());
    CreateSnapshotResponse createSnapshotResponseFirst = client.admin().cluster().prepareCreateSnapshot("test-repo", "test").setWaitForCompletion(true).setIndices("test").get();
    assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponseFirst.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseFirst.getSnapshotInfo().totalShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    {
        SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test").get().getSnapshots().get(0);
        List<SnapshotIndexShardStatus> shards = snapshotStatus.getShards();
        for (SnapshotIndexShardStatus status : shards) {
            assertThat(status.getStats().getProcessedFiles(), greaterThan(1));
        }
    }
    CreateSnapshotResponse createSnapshotResponseSecond = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-1").setWaitForCompletion(true).setIndices("test").get();
    assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponseSecond.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseSecond.getSnapshotInfo().totalShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-1").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    {
        SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-1").get().getSnapshots().get(0);
        List<SnapshotIndexShardStatus> shards = snapshotStatus.getShards();
        for (SnapshotIndexShardStatus status : shards) {
            assertThat(status.getStats().getProcessedFiles(), equalTo(0));
        }
    }
    client().prepareDelete("test", "doc", "1").get();
    CreateSnapshotResponse createSnapshotResponseThird = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-2").setWaitForCompletion(true).setIndices("test").get();
    assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), greaterThan(0));
    assertThat(createSnapshotResponseThird.getSnapshotInfo().successfulShards(), equalTo(createSnapshotResponseThird.getSnapshotInfo().totalShards()));
    assertThat(client.admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-2").get().getSnapshots().get(0).state(), equalTo(SnapshotState.SUCCESS));
    {
        SnapshotStatus snapshotStatus = client.admin().cluster().prepareSnapshotStatus("test-repo").setSnapshots("test-2").get().getSnapshots().get(0);
        List<SnapshotIndexShardStatus> shards = snapshotStatus.getShards();
        for (SnapshotIndexShardStatus status : shards) {
            // we flush before the snapshot such that we have to process the segments_N files plus the .del file
            assertThat(status.getStats().getProcessedFiles(), equalTo(2));
        }
    }
}

Example 32 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class SharedClusterSnapshotRestoreIT method testBatchingShardUpdateTask.

public void testBatchingShardUpdateTask() throws Exception {
    final Client client = client();
    logger.info("-->  creating repository");
    assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    assertAcked(prepareCreate("test-idx", 0, Settings.builder().put("number_of_shards", between(1, 10)).put("number_of_replicas", 0)));
    ensureGreen();
    logger.info("--> indexing some data");
    final int numdocs = randomIntBetween(10, 100);
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test-idx", "type1", Integer.toString(i)).setSource("field1", "bar " + i);
    }
    indexRandom(true, builders);
    flushAndRefresh();
    final int numberOfShards = getNumShards("test-idx").numPrimaries;
    logger.info("number of shards: {}", numberOfShards);
    final ClusterService clusterService = internalCluster().clusterService(internalCluster().getMasterName());
    BlockingClusterStateListener snapshotListener = new BlockingClusterStateListener(clusterService, "update_snapshot [", "update snapshot state", Priority.HIGH);
    try {
        clusterService.addListener(snapshotListener);
        logger.info("--> snapshot");
        ListenableActionFuture<CreateSnapshotResponse> snapshotFuture = client.admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(true).setIndices("test-idx").execute();
        // Await until shard updates are in pending state.
        assertBusyPendingTasks("update snapshot state", numberOfShards);
        snapshotListener.unblock();
        // Check that the snapshot was successful
        CreateSnapshotResponse createSnapshotResponse = snapshotFuture.actionGet();
        assertEquals(SnapshotState.SUCCESS, createSnapshotResponse.getSnapshotInfo().state());
        assertEquals(numberOfShards, createSnapshotResponse.getSnapshotInfo().totalShards());
        assertEquals(numberOfShards, createSnapshotResponse.getSnapshotInfo().successfulShards());
    } finally {
        clusterService.removeListener(snapshotListener);
    }
    // Check that we didn't timeout
    assertFalse(snapshotListener.timedOut());
    // Check that cluster state update task was called only once
    assertEquals(1, snapshotListener.count());
}

Example 33 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class DedicatedClusterSnapshotRestoreIT method testMasterShutdownDuringSnapshot.

public void testMasterShutdownDuringSnapshot() throws Exception {
    Settings masterSettings = Settings.builder().put(Node.NODE_DATA_SETTING.getKey(), false).build();
    Settings dataSettings = Settings.builder().put(Node.NODE_MASTER_SETTING.getKey(), false).build();
    logger.info("-->  starting two master nodes and two data nodes");
    internalCluster().startNode(masterSettings);
    internalCluster().startNode(masterSettings);
    internalCluster().startNode(dataSettings);
    internalCluster().startNode(dataSettings);
    final Client client = client();
    logger.info("-->  creating repository");
    assertAcked(client.admin().cluster().preparePutRepository("test-repo").setType("fs").setSettings(Settings.builder().put("location", randomRepoPath()).put("compress", randomBoolean()).put("chunk_size", randomIntBetween(100, 1000), ByteSizeUnit.BYTES)));
    assertAcked(prepareCreate("test-idx", 0, Settings.builder().put("number_of_shards", between(1, 20)).put("number_of_replicas", 0)));
    ensureGreen();
    logger.info("--> indexing some data");
    final int numdocs = randomIntBetween(10, 100);
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numdocs];
    for (int i = 0; i < builders.length; i++) {
        builders[i] = client().prepareIndex("test-idx", "type1", Integer.toString(i)).setSource("field1", "bar " + i);
    }
    indexRandom(true, builders);
    flushAndRefresh();
    final int numberOfShards = getNumShards("test-idx").numPrimaries;
    logger.info("number of shards: {}", numberOfShards);
    final ClusterService clusterService = internalCluster().clusterService(internalCluster().getMasterName());
    BlockingClusterStateListener snapshotListener = new BlockingClusterStateListener(clusterService, "update_snapshot [", "update snapshot state", Priority.HIGH);
    try {
        clusterService.addListener(snapshotListener);
        logger.info("--> snapshot");
        dataNodeClient().admin().cluster().prepareCreateSnapshot("test-repo", "test-snap").setWaitForCompletion(false).setIndices("test-idx").get();
        // Await until some updates are in pending state.
        assertBusyPendingTasks("update snapshot state", 1);
        logger.info("--> stopping master node");
        internalCluster().stopCurrentMasterNode();
        logger.info("--> unblocking snapshot execution");
        snapshotListener.unblock();
    } finally {
        clusterService.removeListener(snapshotListener);
    }
    logger.info("--> wait until the snapshot is done");
    assertBusy(new Runnable() {

        @Override
        public void run() {
            GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
            SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
            assertTrue(snapshotInfo.state().completed());
        }
    }, 1, TimeUnit.MINUTES);
    logger.info("--> verify that snapshot was succesful");
    GetSnapshotsResponse snapshotsStatusResponse = client().admin().cluster().prepareGetSnapshots("test-repo").setSnapshots("test-snap").get();
    SnapshotInfo snapshotInfo = snapshotsStatusResponse.getSnapshots().get(0);
    assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
    assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());
    assertEquals(0, snapshotInfo.failedShards());
}

Example 34 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class SimpleThreadPoolIT method testThreadNames.

public void testThreadNames() throws Exception {
    ThreadMXBean threadBean = ManagementFactory.getThreadMXBean();
    Set<String> preNodeStartThreadNames = new HashSet<>();
    for (long l : threadBean.getAllThreadIds()) {
        ThreadInfo threadInfo = threadBean.getThreadInfo(l);
        if (threadInfo != null) {
            preNodeStartThreadNames.add(threadInfo.getThreadName());
        }
    }
    logger.info("pre node threads are {}", preNodeStartThreadNames);
    String node = internalCluster().startNode();
    logger.info("do some indexing, flushing, optimize, and searches");
    int numDocs = randomIntBetween(2, 100);
    IndexRequestBuilder[] builders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < numDocs; ++i) {
        builders[i] = client().prepareIndex("idx", "type").setSource(jsonBuilder().startObject().field("str_value", "s" + i).array("str_values", new String[] { "s" + (i * 2), "s" + (i * 2 + 1) }).field("l_value", i).array("l_values", new int[] { i * 2, i * 2 + 1 }).field("d_value", i).array("d_values", new double[] { i * 2, i * 2 + 1 }).endObject());
    }
    indexRandom(true, builders);
    int numSearches = randomIntBetween(2, 100);
    for (int i = 0; i < numSearches; i++) {
        assertNoFailures(client().prepareSearch("idx").setQuery(QueryBuilders.termQuery("str_value", "s" + i)).get());
        assertNoFailures(client().prepareSearch("idx").setQuery(QueryBuilders.termQuery("l_value", i)).get());
    }
    Set<String> threadNames = new HashSet<>();
    for (long l : threadBean.getAllThreadIds()) {
        ThreadInfo threadInfo = threadBean.getThreadInfo(l);
        if (threadInfo != null) {
            threadNames.add(threadInfo.getThreadName());
        }
    }
    logger.info("post node threads are {}", threadNames);
    threadNames.removeAll(preNodeStartThreadNames);
    logger.info("post node *new* threads are {}", threadNames);
    for (String threadName : threadNames) {
        // or the ones that are occasionally come up from ESSingleNodeTestCase
        if (// TODO: this can't possibly be right! single node and integ test are unrelated!
        threadName.contains("[node_s_0]") || threadName.contains("Keep-Alive-Timer")) {
            continue;
        }
        String nodePrefix = "(" + Pattern.quote(InternalTestCluster.TRANSPORT_CLIENT_PREFIX) + ")?(" + Pattern.quote(ESIntegTestCase.SUITE_CLUSTER_NODE_PREFIX) + "|" + Pattern.quote(ESIntegTestCase.TEST_CLUSTER_NODE_PREFIX) + "|" + Pattern.quote("node_tribe2") + ")";
        assertThat(threadName, RegexMatcher.matches("\\[" + nodePrefix + "\\d+\\]"));
    }
}

Example 35 with IndexRequestBuilder

use of org.elasticsearch.action.index.IndexRequestBuilder in project elasticsearch by elastic.

the class MultiSearchTemplateIT method testBasic.

public void testBasic() throws Exception {
    createIndex("msearch");
    final int numDocs = randomIntBetween(10, 100);
    IndexRequestBuilder[] indexRequestBuilders = new IndexRequestBuilder[numDocs];
    for (int i = 0; i < numDocs; i++) {
        indexRequestBuilders[i] = client().prepareIndex("msearch", "test", String.valueOf(i)).setSource("odd", (i % 2 == 0), "group", (i % 3));
    }
    indexRandom(true, indexRequestBuilders);
    final String template = jsonBuilder().startObject().startObject("query").startObject("{{query_type}}").field("{{field_name}}", "{{field_value}}").endObject().endObject().endObject().string();
    MultiSearchTemplateRequest multiRequest = new MultiSearchTemplateRequest();
    // Search #1
    SearchTemplateRequest search1 = new SearchTemplateRequest();
    search1.setRequest(new SearchRequest("msearch"));
    search1.setScriptType(ScriptType.INLINE);
    search1.setScript(template);
    Map<String, Object> params1 = new HashMap<>();
    params1.put("query_type", "match");
    params1.put("field_name", "odd");
    params1.put("field_value", true);
    search1.setScriptParams(params1);
    multiRequest.add(search1);
    // Search #2 (Simulate is true)
    SearchTemplateRequest search2 = new SearchTemplateRequest();
    search2.setRequest(new SearchRequest("msearch"));
    search2.setScriptType(ScriptType.INLINE);
    search2.setScript(template);
    search2.setSimulate(true);
    Map<String, Object> params2 = new HashMap<>();
    params2.put("query_type", "match_phrase_prefix");
    params2.put("field_name", "message");
    params2.put("field_value", "quick brown f");
    search2.setScriptParams(params2);
    multiRequest.add(search2);
    // Search #3
    SearchTemplateRequest search3 = new SearchTemplateRequest();
    search3.setRequest(new SearchRequest("msearch"));
    search3.setScriptType(ScriptType.INLINE);
    search3.setScript(template);
    search3.setSimulate(false);
    Map<String, Object> params3 = new HashMap<>();
    params3.put("query_type", "term");
    params3.put("field_name", "odd");
    params3.put("field_value", "false");
    search3.setScriptParams(params3);
    multiRequest.add(search3);
    // Search #4 (Fail because of unknown index)
    SearchTemplateRequest search4 = new SearchTemplateRequest();
    search4.setRequest(new SearchRequest("unknown"));
    search4.setScriptType(ScriptType.INLINE);
    search4.setScript(template);
    Map<String, Object> params4 = new HashMap<>();
    params4.put("query_type", "match");
    params4.put("field_name", "group");
    params4.put("field_value", "test");
    search4.setScriptParams(params4);
    multiRequest.add(search4);
    // Search #5 (Simulate is true)
    SearchTemplateRequest search5 = new SearchTemplateRequest();
    search5.setRequest(new SearchRequest("msearch"));
    search5.setScriptType(ScriptType.INLINE);
    search5.setScript("{{! ignore me }}{\"query\":{\"terms\":{\"group\":[{{#groups}}{{.}},{{/groups}}]}}}");
    search5.setSimulate(true);
    Map<String, Object> params5 = new HashMap<>();
    params5.put("groups", Arrays.asList(1, 2, 3));
    search5.setScriptParams(params5);
    multiRequest.add(search5);
    MultiSearchTemplateResponse response = client().execute(MultiSearchTemplateAction.INSTANCE, multiRequest).get();
    assertThat(response.getResponses(), arrayWithSize(5));
    MultiSearchTemplateResponse.Item response1 = response.getResponses()[0];
    assertThat(response1.isFailure(), is(false));
    SearchTemplateResponse searchTemplateResponse1 = response1.getResponse();
    assertThat(searchTemplateResponse1.hasResponse(), is(true));
    assertHitCount(searchTemplateResponse1.getResponse(), (numDocs / 2) + (numDocs % 2));
    assertThat(searchTemplateResponse1.getSource().utf8ToString(), equalTo("{\"query\":{\"match\":{\"odd\":\"true\"}}}"));
    MultiSearchTemplateResponse.Item response2 = response.getResponses()[1];
    assertThat(response2.isFailure(), is(false));
    SearchTemplateResponse searchTemplateResponse2 = response2.getResponse();
    assertThat(searchTemplateResponse2.hasResponse(), is(false));
    assertThat(searchTemplateResponse2.getSource().utf8ToString(), equalTo("{\"query\":{\"match_phrase_prefix\":{\"message\":\"quick brown f\"}}}"));
    MultiSearchTemplateResponse.Item response3 = response.getResponses()[2];
    assertThat(response3.isFailure(), is(false));
    SearchTemplateResponse searchTemplateResponse3 = response3.getResponse();
    assertThat(searchTemplateResponse3.hasResponse(), is(true));
    assertHitCount(searchTemplateResponse3.getResponse(), (numDocs / 2));
    assertThat(searchTemplateResponse3.getSource().utf8ToString(), equalTo("{\"query\":{\"term\":{\"odd\":\"false\"}}}"));
    MultiSearchTemplateResponse.Item response4 = response.getResponses()[3];
    assertThat(response4.isFailure(), is(true));
    assertThat(response4.getFailure(), instanceOf(IndexNotFoundException.class));
    assertThat(response4.getFailure().getMessage(), equalTo("no such index"));
    MultiSearchTemplateResponse.Item response5 = response.getResponses()[4];
    assertThat(response5.isFailure(), is(false));
    SearchTemplateResponse searchTemplateResponse5 = response5.getResponse();
    assertThat(searchTemplateResponse5.hasResponse(), is(false));
    assertThat(searchTemplateResponse5.getSource().utf8ToString(), equalTo("{\"query\":{\"terms\":{\"group\":[1,2,3,]}}}"));
}