Examples with BulkRequestBuilder org.elasticsearch.action.bulk.BulkRequestBuilder used on opensource projects

Example 26 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project fess by codelibs.

the class EsAbstractBehavior method delegateBatchRequest.

protected <BUILDER> int[] delegateBatchRequest(final List<? extends Entity> entityList, Function<EsAbstractEntity, BUILDER> call) {
    @SuppressWarnings("unchecked") final BulkList<? extends Entity, BUILDER> bulkList = (BulkList<? extends Entity, BUILDER>) entityList;
    final RequestOptionCall<BUILDER> builderEntityCall = bulkList.getEntityCall();
    final BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (final Entity entity : entityList) {
        final EsAbstractEntity esEntity = (EsAbstractEntity) entity;
        BUILDER builder = call.apply(esEntity);
        if (builder instanceof IndexRequestBuilder) {
            if (builderEntityCall != null) {
                builderEntityCall.callback(builder);
            }
            bulkBuilder.add((IndexRequestBuilder) builder);
        } else if (builder instanceof UpdateRequestBuilder) {
            if (builderEntityCall != null) {
                builderEntityCall.callback(builder);
            }
            bulkBuilder.add((UpdateRequestBuilder) builder);
        } else if (builder instanceof DeleteRequestBuilder) {
            if (builderEntityCall != null) {
                builderEntityCall.callback(builder);
            }
            bulkBuilder.add((DeleteRequestBuilder) builder);
        }
    }
    final RequestOptionCall<BulkRequestBuilder> builderCall = bulkList.getCall();
    if (builderCall != null) {
        builderCall.callback(bulkBuilder);
    }
    final BulkResponse response = bulkBuilder.execute().actionGet(bulkTimeout);
    final BulkItemResponse[] itemResponses = response.getItems();
    if (itemResponses.length != entityList.size()) {
        throw new IllegalStateException("Invalid response size: " + itemResponses.length + " != " + entityList.size());
    }
    final int[] results = new int[itemResponses.length];
    for (int i = 0; i < itemResponses.length; i++) {
        final BulkItemResponse itemResponse = itemResponses[i];
        final Entity entity = entityList.get(i);
        if (entity instanceof EsAbstractEntity) {
            ((EsAbstractEntity) entity).asDocMeta().id(itemResponse.getId());
        }
        results[i] = itemResponse.isFailed() ? 0 : 1;
    }
    return results;
}

Example 27 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project storm by apache.

the class EsState method updateState.

/**
     * Store current state to ElasticSearch.
     *
     * @param tuples list of tuples for storing to ES.
     *               Each tuple should have relevant fields (source, index, type, id) for EsState's tupleMapper to extract ES document.
     */
public void updateState(List<TridentTuple> tuples) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (TridentTuple tuple : tuples) {
        String source = tupleMapper.getSource(tuple);
        String index = tupleMapper.getIndex(tuple);
        String type = tupleMapper.getType(tuple);
        String id = tupleMapper.getId(tuple);
        bulkRequest.add(client.prepareIndex(index, type, id).setSource(source));
    }
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    if (bulkResponse.hasFailures()) {
        LOG.warn("failed processing bulk index requests " + bulkResponse.buildFailureMessage());
        throw new FailedException();
    }
}

Example 28 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class InternalEngineMergeIT method testMergesHappening.

@TestLogging("_root:DEBUG")
public void testMergesHappening() throws InterruptedException, IOException, ExecutionException {
    final int numOfShards = randomIntBetween(1, 5);
    // some settings to keep num segments low
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, numOfShards).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).build()));
    long id = 0;
    final int rounds = scaledRandomIntBetween(50, 300);
    logger.info("Starting rounds [{}] ", rounds);
    for (int i = 0; i < rounds; ++i) {
        final int numDocs = scaledRandomIntBetween(100, 1000);
        BulkRequestBuilder request = client().prepareBulk();
        for (int j = 0; j < numDocs; ++j) {
            request.add(Requests.indexRequest("test").type("type1").id(Long.toString(id++)).source(jsonBuilder().startObject().field("l", randomLong()).endObject()));
        }
        BulkResponse response = request.execute().actionGet();
        refresh();
        assertNoFailures(response);
        IndicesStatsResponse stats = client().admin().indices().prepareStats("test").setSegments(true).setMerge(true).get();
        logger.info("index round [{}] - segments {}, total merges {}, current merge {}", i, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
    }
    final long upperNumberSegments = 2 * numOfShards * 10;
    awaitBusy(() -> {
        IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).setMerge(true).get();
        logger.info("numshards {}, segments {}, total merges {}, current merge {}", numOfShards, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
        long current = stats.getPrimaries().getMerge().getCurrent();
        long count = stats.getPrimaries().getSegments().getCount();
        return count < upperNumberSegments && current == 0;
    });
    IndicesStatsResponse stats = client().admin().indices().prepareStats().setSegments(true).setMerge(true).get();
    logger.info("numshards {}, segments {}, total merges {}, current merge {}", numOfShards, stats.getPrimaries().getSegments().getCount(), stats.getPrimaries().getMerge().getTotal(), stats.getPrimaries().getMerge().getCurrent());
    long count = stats.getPrimaries().getSegments().getCount();
    assertThat(count, Matchers.lessThanOrEqualTo(upperNumberSegments));
}

Example 29 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class RetryTests method setUp.

@Before
public void setUp() throws Exception {
    super.setUp();
    createIndex("source");
    // Build the test data. Don't use indexRandom because that won't work consistently with such small thread pools.
    BulkRequestBuilder bulk = client().prepareBulk();
    for (int i = 0; i < DOC_COUNT; i++) {
        bulk.add(client().prepareIndex("source", "test").setSource("foo", "bar " + i));
    }
    Retry retry = Retry.on(EsRejectedExecutionException.class).policy(BackoffPolicy.exponentialBackoff());
    BulkResponse response = retry.withSyncBackoff(client(), bulk.request());
    assertFalse(response.buildFailureMessage(), response.hasFailures());
    client().admin().indices().prepareRefresh("source").get();
}

Example 30 with BulkRequestBuilder

use of org.elasticsearch.action.bulk.BulkRequestBuilder in project elasticsearch by elastic.

the class ShardInfoIT method testBulkWithIndexAndDeleteItems.

public void testBulkWithIndexAndDeleteItems() throws Exception {
    prepareIndex(1);
    BulkRequestBuilder bulkRequestBuilder = client().prepareBulk();
    for (int i = 0; i < 10; i++) {
        bulkRequestBuilder.add(client().prepareIndex("idx", "type").setSource("{}", XContentType.JSON));
    }
    BulkResponse bulkResponse = bulkRequestBuilder.get();
    bulkRequestBuilder = client().prepareBulk();
    for (BulkItemResponse item : bulkResponse) {
        assertThat(item.isFailed(), equalTo(false));
        assertShardInfo(item.getResponse());
        bulkRequestBuilder.add(client().prepareDelete("idx", "type", item.getId()));
    }
    bulkResponse = bulkRequestBuilder.get();
    for (BulkItemResponse item : bulkResponse) {
        assertThat(item.isFailed(), equalTo(false));
        assertShardInfo(item.getResponse());
    }
}