Examples with BulkResponse org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse used on opensource projects

Example 36 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project fess-crawler by codelibs.

the class EsUrlQueueService method poll.

@Override
public EsUrlQueue poll(final String sessionId) {
    final QueueHolder queueHolder = getQueueHolder(sessionId);
    final Queue<EsUrlQueue> waitingQueue = queueHolder.waitingQueue;
    final Queue<EsUrlQueue> crawlingQueue = queueHolder.crawlingQueue;
    EsUrlQueue urlQueue = waitingQueue.poll();
    if (urlQueue != null) {
        if (crawlingQueue.size() > maxCrawlingQueueSize) {
            crawlingQueue.poll();
        }
        crawlingQueue.add(urlQueue);
        return urlQueue;
    }
    synchronized (queueHolder) {
        urlQueue = waitingQueue.poll();
        if (urlQueue == null) {
            final List<EsUrlQueue> urlQueueList = getList(EsUrlQueue.class, sessionId, null, 0, pollingFetchSize, SortBuilders.fieldSort(CREATE_TIME).order(SortOrder.ASC));
            if (urlQueueList.isEmpty()) {
                return null;
            }
            if (logger.isDebugEnabled()) {
                logger.debug("Queued URL: {}", urlQueueList);
            }
            waitingQueue.addAll(urlQueueList);
            if (!urlQueueList.isEmpty()) {
                try {
                    // delete from es
                    final BulkResponse response = getClient().get(c -> {
                        final BulkRequestBuilder bulkBuilder = c.prepareBulk();
                        for (final EsUrlQueue uq : urlQueueList) {
                            bulkBuilder.add(c.prepareDelete(index, type, uq.getId()));
                        }
                        return bulkBuilder.setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute();
                    });
                    if (response.hasFailures()) {
                        logger.warn(response.buildFailureMessage());
                    }
                } catch (final Exception e) {
                    throw new EsAccessException("Failed to delete " + urlQueueList, e);
                }
            }
            urlQueue = waitingQueue.poll();
            if (urlQueue == null) {
                return null;
            }
        }
    }
    if (crawlingQueue.size() > maxCrawlingQueueSize) {
        crawlingQueue.poll();
    }
    crawlingQueue.add(urlQueue);
    return urlQueue;
}

Example 37 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project samza by apache.

the class ElasticsearchSystemProducer method register.

@Override
public void register(final String source) {
    BulkProcessor.Listener listener = new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
        // Nothing to do.
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            boolean hasFatalError = false;
            // Do not consider version conficts to be errors. Ignore old versions
            if (response.hasFailures()) {
                for (BulkItemResponse itemResp : response.getItems()) {
                    if (itemResp.isFailed()) {
                        if (itemResp.getFailure().getStatus().equals(RestStatus.CONFLICT)) {
                            LOGGER.info("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage());
                        } else {
                            hasFatalError = true;
                            LOGGER.error("Failed to index document in Elasticsearch: " + itemResp.getFailureMessage());
                        }
                    }
                }
            }
            if (hasFatalError) {
                sendFailed.set(true);
            } else {
                updateSuccessMetrics(response);
            }
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOGGER.error(failure.getMessage());
            thrown.compareAndSet(null, failure);
            sendFailed.set(true);
        }

        private void updateSuccessMetrics(BulkResponse response) {
            metrics.bulkSendSuccess.inc();
            int writes = 0;
            for (BulkItemResponse itemResp : response.getItems()) {
                if (itemResp.isFailed()) {
                    if (itemResp.getFailure().getStatus().equals(RestStatus.CONFLICT)) {
                        metrics.conflicts.inc();
                    }
                } else {
                    ActionResponse resp = itemResp.getResponse();
                    if (resp instanceof IndexResponse) {
                        writes += 1;
                        if (((IndexResponse) resp).isCreated()) {
                            metrics.inserts.inc();
                        } else {
                            metrics.updates.inc();
                        }
                    } else {
                        LOGGER.error("Unexpected Elasticsearch action response type: " + resp.getClass().getSimpleName());
                    }
                }
            }
            LOGGER.info(String.format("Wrote %s messages from %s to %s.", writes, source, system));
        }
    };
    sourceBulkProcessor.put(source, bulkProcessorFactory.getBulkProcessor(client, listener));
}

Example 38 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project samza by apache.

the class ElasticsearchSystemProducerTest method testIgnoreVersionConficts.

@Test
public void testIgnoreVersionConficts() throws Exception {
    ArgumentCaptor<BulkProcessor.Listener> listenerCaptor = ArgumentCaptor.forClass(BulkProcessor.Listener.class);
    when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture())).thenReturn(processorOne);
    producer.register(SOURCE_ONE);
    BulkResponse response = getRespWithFailedDocument(RestStatus.CONFLICT);
    listenerCaptor.getValue().afterBulk(0, null, response);
    assertEquals(1, metrics.conflicts.getCount());
    producer.flush(SOURCE_ONE);
}

Example 39 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project flink by apache.

the class Elasticsearch7SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 40 with BulkResponse

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkResponse in project graylog2-server by Graylog2.

the class FixtureImporterES7 method importNode.

private void importNode(JsonNode root) throws IOException {
    /* This supports the nosqlunit DataSet structure:
         *
         *  {
         *    "documents": [
         *      {
         *        "document": [
         *          {
         *            "index": {
         *              "indexName": "graylog_0",
         *              "indexId": "0"
         *            }
         *          },
         *          {
         *            "data": {
         *              "source": "example.org",
         *              "message": "Hi",
         *              "timestamp": "2015-01-01 01:00:00.000"
         *            }
         *          }
         *        ]
         *      }
         *    ]
         *  }
         */
    final BulkRequest bulkRequest = new BulkRequest();
    final Set<String> targetIndices = new HashSet<>();
    for (final JsonNode document : root.path("documents")) {
        final List<JsonNode> indexes = new ArrayList<>();
        Map<String, Object> data = new HashMap<>();
        for (JsonNode entry : document.path("document")) {
            if (entry.hasNonNull("index")) {
                indexes.add(entry.path("index"));
            } else if (entry.hasNonNull("data")) {
                data = OBJECT_MAPPER.convertValue(entry.path("data"), TypeReferences.MAP_STRING_OBJECT);
            }
        }
        for (final JsonNode index : indexes) {
            final IndexRequest indexRequest = new IndexRequest().source(data);
            final String indexName = index.path("indexName").asText(null);
            if (indexName == null) {
                throw new IllegalArgumentException("Missing indexName in " + index);
            }
            targetIndices.add(indexName);
            indexRequest.index(indexName);
            if (index.hasNonNull("indexId")) {
                indexRequest.id(index.path("indexId").asText());
            }
            bulkRequest.add(indexRequest);
        }
    }
    for (String indexName : targetIndices) {
        if (!indexExists(indexName)) {
            createIndex(indexName);
        }
    }
    final BulkResponse result = client.execute((c, requestOptions) -> c.bulk(bulkRequest, requestOptions), "Unable to import fixtures.");
    if (result.hasFailures()) {
        throw new IllegalStateException("Error while bulk indexing documents: " + result.buildFailureMessage());
    }
}