Examples with BulkResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 51 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project pentaho-kettle by pentaho.

the class ElasticSearchBulk method processBatch.

private boolean processBatch(boolean makeNew) throws KettleStepException {
    ActionFuture<BulkResponse> actionFuture = currentRequest.execute();
    boolean responseOk = false;
    BulkResponse response = null;
    try {
        if (timeout != null && timeoutUnit != null) {
            response = actionFuture.actionGet(timeout, timeoutUnit);
        } else {
            response = actionFuture.actionGet();
        }
    } catch (ElasticsearchException e) {
        String msg = BaseMessages.getString(PKG, "ElasticSearchBulk.Error.BatchExecuteFail", e.getLocalizedMessage());
        if (e instanceof ElasticsearchTimeoutException) {
            msg = BaseMessages.getString(PKG, "ElasticSearchBulk.Error.Timeout");
        }
        logError(msg);
        rejectAllRows(msg);
    }
    if (response != null) {
        responseOk = handleResponse(response);
        requestsBuffer.clear();
    } else {
        // have to assume all failed
        numberOfErrors += currentRequest.numberOfActions();
        setErrors(numberOfErrors);
    }
    if (makeNew) {
        currentRequest = client.prepareBulk();
        data.nextBufferRowIdx = 0;
        data.inputRowBuffer = new Object[batchSize][];
    } else {
        currentRequest = null;
        data.inputRowBuffer = null;
    }
    return responseOk;
}

Example 52 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project datashare by ICIJ.

the class ElasticsearchIndexer method executeBulk.

private boolean executeBulk(BulkRequest bulkRequest) throws IOException {
    bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);
    BulkResponse bulkResponse = client.bulk(bulkRequest, RequestOptions.DEFAULT);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse resp : bulkResponse.getItems()) {
            if (resp.isFailed()) {
                LOGGER.error("bulk request failed : {}", resp.getFailureMessage());
            }
        }
        return false;
    }
    return true;
}

Example 53 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class InternalEngineMergeIT method testMergesHappening.

@TestLogging("_root:DEBUG")
public void testMergesHappening() throws InterruptedException, IOException, ExecutionException {
    final int numOfShards = randomIntBetween(1, 5);
    // some settings to keep num segments low
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numOfShards).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build()));
    long id = 0;
    final int rounds = scaledRandomIntBetween(50, 300);
    logger.info("Starting rounds [{}] ", rounds);
    for (int i = 0; i < rounds; ++i) {
        final int numDocs = scaledRandomIntBetween(100, 1000);
        BulkRequestBuilder request = client().prepareBulk();
        for (int j = 0; j < numDocs; ++j) {
            request.add(Requests.indexRequest("test").type("type1").id(Long.toString(id++)).source(jsonBuilder().startObject().field("l", randomLong()).endObject()));
        }
        BulkResponse response = request.execute().actionGet();
        refresh();
        assertNoFailures(response);
        IndicesStatsResponse stats = client().admin().indices().prepareStats("test").setSegments(true).setMerge(true).get();
        logger.info("index round [{}] - segments {}, total merges {}, current merge {}", i, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
    }
    final long upperNumberSegments = 2 * numOfShards * 10;
    awaitBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).setMerge(true).get();
        logger.info("numshards {}, segments {}, total merges {}, current merge {}", numOfShards, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
        long current = stats.getPrimaries().getMerge().getCurrent();
        long count = stats.getPrimaries().getSegments().getCount();
        return count < upperNumberSegments && current == 0;
    });
    IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).setMerge(true).get();
    logger.info("numshards {}, segments {}, total merges {}, current merge {}", numOfShards, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
    long count = stats.getPrimaries().getSegments().getCount();
    assertThat(count, Matchers.lessThanOrEqualTo(upperNumberSegments));
}

Example 54 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class RetryTests method setUp.

@Before
public void setUp() throws Exception {
    super.setUp();
    createIndex("source");
    // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools.
    BulkRequestBuilder bulk = client().prepareBulk();
    for (int i = 0; i < DOC_COUNT; i++) {
        bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i));
    }
    Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff());
    BulkResponse response = retry.withSyncBackoff(client(), bulk.request());
    assertFalse(response.buildFailureMessage(), response.hasFailures());
    client().admin().indices().prepareRefresh("source").get();
}

Example 55 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project elasticsearch by elastic.

the class DocumentActionsIT method testBulk.

public void testBulk() throws Exception {
    createIndex();
    NumShards numShards = getNumShards(getConcreteIndexName());
    logger.info("-> running Cluster Health");
    ensureGreen();
    BulkResponse bulkResponse = client().prepareBulk().add(client().prepareIndex().setIndex("test").setType("type1").setId("1").setSource(source("1", "test"))).add(client().prepareIndex().setIndex("test").setType("type1").setId("2").setSource(source("2", "test")).setCreate(true)).add(client().prepareIndex().setIndex("test").setType("type1").setSource(source("3", "test"))).add(client().prepareDelete().setIndex("test").setType("type1").setId("1")).add(// failure
    client().prepareIndex().setIndex("test").setType("type1").setSource("{ xxx }", XContentType.JSON)).execute().actionGet();
    assertThat(bulkResponse.hasFailures(), equalTo(true));
    assertThat(bulkResponse.getItems().length, equalTo(5));
    assertThat(bulkResponse.getItems()[0].isFailed(), equalTo(false));
    assertThat(bulkResponse.getItems()[0].getOpType(), equalTo(OpType.INDEX));
    assertThat(bulkResponse.getItems()[0].getIndex(), equalTo(getConcreteIndexName()));
    assertThat(bulkResponse.getItems()[0].getType(), equalTo("type1"));
    assertThat(bulkResponse.getItems()[0].getId(), equalTo("1"));
    assertThat(bulkResponse.getItems()[1].isFailed(), equalTo(false));
    assertThat(bulkResponse.getItems()[1].getOpType(), equalTo(OpType.CREATE));
    assertThat(bulkResponse.getItems()[1].getIndex(), equalTo(getConcreteIndexName()));
    assertThat(bulkResponse.getItems()[1].getType(), equalTo("type1"));
    assertThat(bulkResponse.getItems()[1].getId(), equalTo("2"));
    assertThat(bulkResponse.getItems()[2].isFailed(), equalTo(false));
    assertThat(bulkResponse.getItems()[2].getOpType(), equalTo(OpType.INDEX));
    assertThat(bulkResponse.getItems()[2].getIndex(), equalTo(getConcreteIndexName()));
    assertThat(bulkResponse.getItems()[2].getType(), equalTo("type1"));
    String generatedId3 = bulkResponse.getItems()[2].getId();
    assertThat(bulkResponse.getItems()[3].isFailed(), equalTo(false));
    assertThat(bulkResponse.getItems()[3].getOpType(), equalTo(OpType.DELETE));
    assertThat(bulkResponse.getItems()[3].getIndex(), equalTo(getConcreteIndexName()));
    assertThat(bulkResponse.getItems()[3].getType(), equalTo("type1"));
    assertThat(bulkResponse.getItems()[3].getId(), equalTo("1"));
    assertThat(bulkResponse.getItems()[4].isFailed(), equalTo(true));
    assertThat(bulkResponse.getItems()[4].getOpType(), equalTo(OpType.INDEX));
    assertThat(bulkResponse.getItems()[4].getIndex(), equalTo(getConcreteIndexName()));
    assertThat(bulkResponse.getItems()[4].getType(), equalTo("type1"));
    waitForRelocation(ClusterHealthStatus.GREEN);
    RefreshResponse refreshResponse = client().admin().indices().prepareRefresh("test").execute().actionGet();
    assertNoFailures(refreshResponse);
    assertThat(refreshResponse.getSuccessfulShards(), equalTo(numShards.totalNumShards));
    for (int i = 0; i < 5; i++) {
        GetResponse getResult = client().get(getRequest("test").type("type1").id("1")).actionGet();
        assertThat(getResult.getIndex(), equalTo(getConcreteIndexName()));
        assertThat("cycle #" + i, getResult.isExists(), equalTo(false));
        getResult = client().get(getRequest("test").type("type1").id("2")).actionGet();
        assertThat("cycle #" + i, getResult.getSourceAsString(), equalTo(source("2", "test").string()));
        assertThat(getResult.getIndex(), equalTo(getConcreteIndexName()));
        getResult = client().get(getRequest("test").type("type1").id(generatedId3)).actionGet();
        assertThat("cycle #" + i, getResult.getSourceAsString(), equalTo(source("3", "test").string()));
        assertThat(getResult.getIndex(), equalTo(getConcreteIndexName()));
    }
}