Examples with BulkRequest org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest used on opensource projects

Example 21 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project flink by apache.

the class Elasticsearch7SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 22 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project hazelcast by hazelcast.

the class AuthElasticSinksTest method given_clientWithWrongPassword_whenWriteToElasticSink_thenFailWithAuthenticationException.

@Test
public void given_clientWithWrongPassword_whenWriteToElasticSink_thenFailWithAuthenticationException() {
    ElasticsearchContainer container = ElasticSupport.secureElastic.get();
    String containerIp = container.getContainerIpAddress();
    Integer port = container.getMappedPort(PORT);
    Sink<TestItem> elasticSink = new ElasticSinkBuilder<>().clientFn(() -> client("elastic", "WrongPassword", containerIp, port)).bulkRequestFn(() -> new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE)).mapToRequestFn((TestItem item) -> new IndexRequest("my-index").source(item.asMap())).retries(0).build();
    Pipeline p = Pipeline.create();
    p.readFrom(TestSources.items(new TestItem("id", "Frantisek"))).writeTo(elasticSink);
    assertThatThrownBy(() -> submitJob(p)).hasRootCauseInstanceOf(ElasticsearchStatusException.class).hasStackTraceContaining("unable to authenticate user [elastic]");
}

Example 23 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project hazelcast by hazelcast.

the class AuthElasticSinksTest method given_authenticatedClient_whenWriteToElasticSink_thenFinishSuccessfully.

@Test
public void given_authenticatedClient_whenWriteToElasticSink_thenFinishSuccessfully() throws IOException {
    Sink<TestItem> elasticSink = new ElasticSinkBuilder<>().clientFn(elasticClientSupplier()).bulkRequestFn(() -> new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE)).mapToRequestFn((TestItem item) -> new IndexRequest("my-index").source(item.asMap())).build();
    Pipeline p = Pipeline.create();
    p.readFrom(TestSources.items(new TestItem("id", "Frantisek"))).writeTo(elasticSink);
    submitJob(p);
    assertSingleDocument();
}

Example 24 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project hazelcast by hazelcast.

the class CommonElasticSinksTest method given_documentInIndex_whenWriteToElasticSinkDeleteRequestTwice_then_jobShouldFinishSuccessfully.

@Test
public void given_documentInIndex_whenWriteToElasticSinkDeleteRequestTwice_then_jobShouldFinishSuccessfully() throws Exception {
    Map<String, Object> doc = new HashMap<>();
    doc.put("name", "Frantisek");
    String id = indexDocument("my-index", doc);
    Sink<String> elasticSink = new ElasticSinkBuilder<>().clientFn(elasticClientSupplier()).mapToRequestFn((String item) -> new DeleteRequest("my-index", item)).bulkRequestFn(() -> new BulkRequest().setRefreshPolicy(RefreshPolicy.IMMEDIATE)).build();
    Pipeline p = Pipeline.create();
    p.readFrom(TestSources.items(id)).writeTo(elasticSink);
    // Submit job 2x to delete non-existing document on 2nd run
    submitJob(p);
    submitJob(p);
    assertNoDocuments("my-index");
}

Example 25 with BulkRequest

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.bulk.BulkRequest in project graylog2-server by Graylog2.

the class ClientES7 method bulkIndex.

@Override
public void bulkIndex(BulkIndexRequest bulkIndexRequest) {
    final BulkRequest bulkRequest = new BulkRequest();
    bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    bulkIndexRequest.requests().forEach((indexName, documents) -> documents.forEach(doc -> bulkRequest.add(createIndexRequest(indexName, doc))));
    client.execute((c, requestOptions) -> c.bulk(bulkRequest, requestOptions));
}