Examples with BulkProcessor org.elasticsearch.action.bulk.BulkProcessor used on opensource projects

Example 1 with BulkProcessor

use of org.elasticsearch.action.bulk.BulkProcessor in project samza by apache.

the class BulkProcessorFactory method getBulkProcessor.

public BulkProcessor getBulkProcessor(Client client, BulkProcessor.Listener listener) {
    BulkProcessor.Builder builder = BulkProcessor.builder(client, listener);
    // Concurrent requests set to 0 to ensure ordering of documents is maintained in batches.
    // This also means BulkProcessor#flush() is blocking as is also required.
    builder.setConcurrentRequests(0);
    if (config.getBulkFlushMaxActions().isPresent()) {
        builder.setBulkActions(config.getBulkFlushMaxActions().get());
    }
    if (config.getBulkFlushMaxSizeMB().isPresent()) {
        builder.setBulkSize(new ByteSizeValue(config.getBulkFlushMaxSizeMB().get(), ByteSizeUnit.MB));
    }
    if (config.getBulkFlushIntervalMS().isPresent()) {
        builder.setFlushInterval(TimeValue.timeValueMillis(config.getBulkFlushIntervalMS().get()));
    }
    return builder.build();
}

Example 2 with BulkProcessor

use of org.elasticsearch.action.bulk.BulkProcessor in project incubator-sdap-mudrod by apache.

the class ESDriver method createBulkProcessor.

public void createBulkProcessor() {
    LOG.debug("Creating BulkProcessor with maxBulkDocs={}, maxBulkLength={}", 1000, 2500500);
    setBulkProcessor(BulkProcessor.builder(getClient(), new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            LOG.debug("ESDriver#createBulkProcessor @Override #beforeBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            LOG.debug("ESDriver#createBulkProcessor @Override #afterBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOG.error("Bulk request has failed!");
            throw new RuntimeException("Caught exception in bulk: " + request.getDescription() + ", failure: " + failure, failure);
        }
    }).setBulkActions(1000).setBulkSize(new ByteSizeValue(2500500, ByteSizeUnit.GB)).setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10)).setConcurrentRequests(1).build());
}

Example 3 with BulkProcessor

use of org.elasticsearch.action.bulk.BulkProcessor in project flink by apache.

the class ElasticsearchSinkBase method buildBulkProcessor.

/**
	 * Build the {@link BulkProcessor}.
	 *
	 * Note: this is exposed for testing purposes.
	 */
protected BulkProcessor buildBulkProcessor(BulkProcessor.Listener listener) {
    checkNotNull(listener);
    BulkProcessor.Builder bulkProcessorBuilder = BulkProcessor.builder(client, listener);
    // This makes flush() blocking
    bulkProcessorBuilder.setConcurrentRequests(0);
    if (bulkProcessorFlushMaxActions != null) {
        bulkProcessorBuilder.setBulkActions(bulkProcessorFlushMaxActions);
    }
    if (bulkProcessorFlushMaxSizeMb != null) {
        bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, ByteSizeUnit.MB));
    }
    if (bulkProcessorFlushIntervalMillis != null) {
        bulkProcessorBuilder.setFlushInterval(TimeValue.timeValueMillis(bulkProcessorFlushIntervalMillis));
    }
    // if backoff retrying is disabled, bulkProcessorFlushBackoffPolicy will be null
    callBridge.configureBulkProcessorBackoff(bulkProcessorBuilder, bulkProcessorFlushBackoffPolicy);
    return bulkProcessorBuilder.build();
}

Example 4 with BulkProcessor

use of org.elasticsearch.action.bulk.BulkProcessor in project molgenis by molgenis.

the class ClientFacade method processDocumentActions.

public void processDocumentActions(Stream<DocumentAction> documentActions) {
    LOG.trace("Processing document actions ...");
    BulkProcessor bulkProcessor = bulkProcessorFactory.create(client);
    try {
        documentActions.forEachOrdered(documentAction -> {
            DocWriteRequest docWriteRequest = toDocWriteRequest(documentAction);
            bulkProcessor.add(docWriteRequest);
        });
    } finally {
        waitForCompletion(bulkProcessor);
        LOG.debug("Processed document actions.");
    }
}

Example 5 with BulkProcessor

use of org.elasticsearch.action.bulk.BulkProcessor in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method reindex.

private void reindex(final ReindexResponse resp, final String indexName, final String docType) {
    // Only retrieve masters - we'll query for the overrides
    final QueryBuilder qb = getFilters(RecurringRetrievalMode.entityOnly).getAllForReindex(docType);
    // 1 minute
    final int timeoutMillis = 60000;
    final TimeValue tv = new TimeValue(timeoutMillis);
    final int batchSize = 100;
    final Client cl = getClient(resp);
    if (cl == null) {
        return;
    }
    checkUidsMap();
    // Start with default index as source
    targetIndex = Util.buildPath(false, idxpars.getUserIndexName());
    final BulkProcessor bulkProcessor = BulkProcessor.builder(cl, new BulkListener()).setBulkActions(batchSize).setConcurrentRequests(3).setFlushInterval(tv).build();
    SearchResponse scrollResp = cl.prepareSearch(targetIndex).setSearchType(SearchType.SCAN).setScroll(tv).setQuery(qb).setSize(batchSize).execute().actionGet();
    // Switch to new index
    targetIndex = indexName;
    // Scroll until no hits are returned
    while (true) {
        for (final SearchHit hit : scrollResp.getHits().getHits()) {
            final String dtype = hit.getType();
            resp.incProcessed();
            if ((resp.getProcessed() % 250) == 0) {
                info("processed " + docType + ": " + resp.getProcessed());
            }
            if (dtype.equals(docTypeUpdateTracker)) {
                continue;
            }
            resp.getStats().inc(docToType.getOrDefault(dtype, unreachableEntities));
            final ReindexResponse.Failure hitResp = new ReindexResponse.Failure();
            final Object entity = makeEntity(hitResp, hit, null);
            if (entity == null) {
                warn("Unable to build entity " + hit.sourceAsString());
                resp.incTotalFailed();
                if (resp.getTotalFailed() < 50) {
                    resp.addFailure(hitResp);
                }
                continue;
            }
            if (entity instanceof BwShareableDbentity) {
                final BwShareableDbentity ent = (BwShareableDbentity) entity;
                try {
                    principal = BwPrincipal.makePrincipal(ent.getOwnerHref());
                } catch (final CalFacadeException cfe) {
                    errorReturn(resp, cfe);
                    return;
                }
            }
            if (entity instanceof EventInfo) {
                // This might be a single event or a recurring event.
                final EventInfo ei = (EventInfo) entity;
                final BwEvent ev = ei.getEvent();
                if (ev.getRecurring()) {
                    resp.incRecurring();
                }
                if (!reindexEvent(hitResp, indexName, hit, ei, bulkProcessor)) {
                    warn("Unable to iondex event " + hit.sourceAsString());
                    resp.incTotalFailed();
                    if (resp.getTotalFailed() < 50) {
                        resp.addFailure(hitResp);
                    }
                }
            } else {
                final EsDocInfo doc = makeDoc(resp, entity);
                if (doc == null) {
                    if (resp.getStatus() != ok) {
                        resp.addFailure(hitResp);
                    }
                    continue;
                }
                final IndexRequest request = new IndexRequest(indexName, hit.type(), doc.getId());
                request.source(doc.getSource());
                bulkProcessor.add(request);
                if (entity instanceof BwEventProperty) {
                    if (!cacheEvprop(hitResp, (BwEventProperty) entity)) {
                        resp.addFailure(hitResp);
                    }
                }
            }
        }
        scrollResp = cl.prepareSearchScroll(scrollResp.getScrollId()).setScroll(tv).execute().actionGet();
        // Break condition: No hits are returned
        if (scrollResp.getHits().getHits().length == 0) {
            break;
        }
    }
    try {
        bulkProcessor.awaitClose(10, TimeUnit.MINUTES);
    } catch (final InterruptedException e) {
        errorReturn(resp, "Final bulk close was interrupted. Records may be missing", failed);
    }
    if (uidsSet > 0) {
        info("Uids set: " + uidsSet);
        info("uidOverridesSet: " + uidOverridesSet);
    }
    uidsMap = null;
    uidsOverideMap = null;
}