Examples with BulkRequestBuilder org.elasticsearch.action.bulk.BulkRequestBuilder used on opensource projects

Example 6 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class WaitUntilRefreshIT method testBulk.

public void testBulk() {
    // Index by bulk with RefreshPolicy.WAIT_UNTIL
    BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareIndex("test", "test", "1").setSource("foo", "bar"));
    assertBulkSuccess(bulk.get());
    assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");
    // Update by bulk with RefreshPolicy.WAIT_UNTIL
    bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareUpdate("test", "test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz"));
    assertBulkSuccess(bulk.get());
    assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");
    // Delete by bulk with RefreshPolicy.WAIT_UNTIL
    bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareDelete("test", "test", "1"));
    assertBulkSuccess(bulk.get());
    assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get());
    // Update makes a noop
    bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareDelete("test", "test", "1"));
    assertBulkSuccess(bulk.get());
}

Example 7 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}

Example 8 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class NoMasterNodeIT method testNoMasterActions.

public void testNoMasterActions() throws Exception {
    Settings settings = Settings.builder().put("discovery.type", "zen").put("action.auto_create_index", true).put("discovery.zen.minimum_master_nodes", 2).put(ZenDiscovery.PING_TIMEOUT_SETTING.getKey(), "200ms").put("discovery.initial_state_timeout", "500ms").put(DiscoverySettings.NO_MASTER_BLOCK_SETTING.getKey(), "all").build();
    TimeValue timeout = TimeValue.timeValueMillis(200);
    internalCluster().startNode(settings);
    // start a second node, create an index, and then shut it down so we have no master block
    internalCluster().startNode(settings);
    createIndex("test");
    client().admin().cluster().prepareHealth("test").setWaitForGreenStatus().execute().actionGet();
    internalCluster().stopRandomDataNode();
    assertBusy(new Runnable() {

        @Override
        public void run() {
            ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState();
            assertTrue(state.blocks().hasGlobalBlock(DiscoverySettings.NO_MASTER_BLOCK_ID));
        }
    });
    assertThrows(client().prepareGet("test", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    assertThrows(client().prepareGet("no_index", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    assertThrows(client().prepareMultiGet().add("test", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    assertThrows(client().prepareMultiGet().add("no_index", "type1", "1"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    assertThrows(client().admin().indices().prepareAnalyze("test", "this is a test"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    assertThrows(client().admin().indices().prepareAnalyze("no_index", "this is a test"), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    assertThrows(client().prepareSearch("test").setSize(0), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    assertThrows(client().prepareSearch("no_index").setSize(0), ClusterBlockException.class, RestStatus.SERVICE_UNAVAILABLE);
    checkUpdateAction(false, timeout, client().prepareUpdate("test", "type1", "1").setScript(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())).setTimeout(timeout));
    checkUpdateAction(true, timeout, client().prepareUpdate("no_index", "type1", "1").setScript(new Script(ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG, "test script", Collections.emptyMap())).setTimeout(timeout));
    checkWriteAction(false, timeout, client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
    checkWriteAction(true, timeout, client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()).setTimeout(timeout));
    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    bulkRequestBuilder.add(client().prepareIndex("test", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    bulkRequestBuilder.add(client().prepareIndex("test", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    // the request should fail very quickly - use a large timeout and make sure it didn't pass...
    timeout = new TimeValue(5000);
    bulkRequestBuilder.setTimeout(timeout);
    checkWriteAction(false, timeout, bulkRequestBuilder);
    bulkRequestBuilder = client().prepareBulk();
    bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "1").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    bulkRequestBuilder.add(client().prepareIndex("no_index", "type1", "2").setSource(XContentFactory.jsonBuilder().startObject().endObject()));
    bulkRequestBuilder.setTimeout(timeout);
    checkWriteAction(true, timeValueSeconds(5), bulkRequestBuilder);
    internalCluster().startNode(settings);
    client().admin().cluster().prepareHealth().setWaitForGreenStatus().setWaitForNodes("2").execute().actionGet();
}

Example 9 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project sonarqube by SonarSource.

the class BulkIndexer method executeBulk.

private void executeBulk() {
    final BulkRequestBuilder req = this.bulkRequest;
    this.bulkRequest = client.prepareBulk().setRefresh(false);
    semaphore.acquireUninterruptibly();
    req.execute(new BulkResponseActionListener(req));
}

Example 10 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project sonarqube by SonarSource.

the class IssueIndexer method deleteByKeys.

public void deleteByKeys(String projectUuid, List<String> issueKeys) {
    if (issueKeys.isEmpty()) {
        return;
    }
    int count = 0;
    BulkRequestBuilder builder = esClient.prepareBulk();
    for (String issueKey : issueKeys) {
        builder.add(esClient.prepareDelete(INDEX_TYPE_ISSUE, issueKey).setRefresh(false).setRouting(projectUuid));
        count++;
        if (count >= MAX_BATCH_SIZE) {
            EsUtils.executeBulkRequest(builder, DELETE_ERROR_MESSAGE, projectUuid);
            builder = esClient.prepareBulk();
            count = 0;
        }
    }
    EsUtils.executeBulkRequest(builder, DELETE_ERROR_MESSAGE, projectUuid);
    esClient.prepareRefresh(INDEX_TYPE_ISSUE.getIndex()).get();
}