Examples with ByteSizeValue org.elasticsearch.common.unit.ByteSizeValue used on opensource projects

Example 46 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project samza by apache.

the class BulkProcessorFactory method getBulkProcessor.

public BulkProcessor getBulkProcessor(Client client, BulkProcessor.Listener listener) {
    BulkProcessor.Builder builder = BulkProcessor.builder(client, listener);
    // Concurrent requests set to 0 to ensure ordering of documents is maintained in batches.
    // This also means BulkProcessor#flush() is blocking as is also required.
    builder.setConcurrentRequests(0);
    if (config.getBulkFlushMaxActions().isPresent()) {
        builder.setBulkActions(config.getBulkFlushMaxActions().get());
    }
    if (config.getBulkFlushMaxSizeMB().isPresent()) {
        builder.setBulkSize(new ByteSizeValue(config.getBulkFlushMaxSizeMB().get(), ByteSizeUnit.MB));
    }
    if (config.getBulkFlushIntervalMS().isPresent()) {
        builder.setFlushInterval(TimeValue.timeValueMillis(config.getBulkFlushIntervalMS().get()));
    }
    return builder.build();
}

Example 47 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project incubator-sdap-mudrod by apache.

the class ESDriver method createBulkProcessor.

public void createBulkProcessor() {
    LOG.debug("Creating BulkProcessor with maxBulkDocs={}, maxBulkLength={}", 1000, 2500500);
    setBulkProcessor(BulkProcessor.builder(getClient(), new BulkProcessor.Listener() {

        @Override
        public void beforeBulk(long executionId, BulkRequest request) {
            LOG.debug("ESDriver#createBulkProcessor @Override #beforeBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
            LOG.debug("ESDriver#createBulkProcessor @Override #afterBulk is not implemented yet!");
        }

        @Override
        public void afterBulk(long executionId, BulkRequest request, Throwable failure) {
            LOG.error("Bulk request has failed!");
            throw new RuntimeException("Caught exception in bulk: " + request.getDescription() + ", failure: " + failure, failure);
        }
    }).setBulkActions(1000).setBulkSize(new ByteSizeValue(2500500, ByteSizeUnit.GB)).setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10)).setConcurrentRequests(1).build());
}

Example 48 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project flink by apache.

the class Elasticsearch7SinkBuilder method getBulkProcessorBuilderFactory.

@Override
protected BulkProcessorBuilderFactory getBulkProcessorBuilderFactory() {
    return new BulkProcessorBuilderFactory() {

        @Override
        public BulkProcessor.Builder apply(RestHighLevelClient client, BulkProcessorConfig bulkProcessorConfig, BulkProcessor.Listener listener) {
            BulkProcessor.Builder builder = BulkProcessor.builder(new // This cannot be inlined as a
            BulkRequestConsumerFactory() {

                // lambda because then
                // deserialization fails
                @Override
                public void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
                    client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener);
                }
            }, listener);
            if (bulkProcessorConfig.getBulkFlushMaxActions() != -1) {
                builder.setBulkActions(bulkProcessorConfig.getBulkFlushMaxActions());
            }
            if (bulkProcessorConfig.getBulkFlushMaxMb() != -1) {
                builder.setBulkSize(new ByteSizeValue(bulkProcessorConfig.getBulkFlushMaxMb(), ByteSizeUnit.MB));
            }
            if (bulkProcessorConfig.getBulkFlushInterval() != -1) {
                builder.setFlushInterval(new TimeValue(bulkProcessorConfig.getBulkFlushInterval()));
            }
            BackoffPolicy backoffPolicy;
            final TimeValue backoffDelay = new TimeValue(bulkProcessorConfig.getBulkFlushBackOffDelay());
            final int maxRetryCount = bulkProcessorConfig.getBulkFlushBackoffRetries();
            switch(bulkProcessorConfig.getFlushBackoffType()) {
                case CONSTANT:
                    backoffPolicy = BackoffPolicy.constantBackoff(backoffDelay, maxRetryCount);
                    break;
                case EXPONENTIAL:
                    backoffPolicy = BackoffPolicy.exponentialBackoff(backoffDelay, maxRetryCount);
                    break;
                case NONE:
                    backoffPolicy = BackoffPolicy.noBackoff();
                    break;
                default:
                    throw new IllegalArgumentException("Received unknown backoff policy type " + bulkProcessorConfig.getFlushBackoffType());
            }
            builder.setBackoffPolicy(backoffPolicy);
            return builder;
        }
    };
}

Example 49 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project flink by apache.

the class ElasticsearchSinkBase method configureBulkSize.

private void configureBulkSize(BulkProcessor.Builder bulkProcessorBuilder) {
    final ByteSizeUnit sizeUnit;
    if (bulkProcessorFlushMaxSizeMb == -1) {
        // bulk size can be disabled with -1, however the ByteSizeValue constructor accepts -1
        // only with BYTES as the size unit
        sizeUnit = ByteSizeUnit.BYTES;
    } else {
        sizeUnit = ByteSizeUnit.MB;
    }
    bulkProcessorBuilder.setBulkSize(new ByteSizeValue(bulkProcessorFlushMaxSizeMb, sizeUnit));
}

Example 50 with ByteSizeValue

use of org.elasticsearch.common.unit.ByteSizeValue in project elasticsearch-jdbc by jprante.

the class ClientTests method testClient.

@Test
public void testClient() throws IOException {
    // disable DNS caching for failover
    Security.setProperty("networkaddress.cache.ttl", "0");
    Settings settings = Settings.settingsBuilder().putArray("elasticsearch.host", new String[] { "localhost:9300", "localhost:9301" }).put("transport.type", "org.elasticsearch.transport.netty.FoundNettyTransport").put("transport.found.ssl-ports", 9443).build();
    Integer maxbulkactions = settings.getAsInt("max_bulk_actions", 10000);
    Integer maxconcurrentbulkrequests = settings.getAsInt("max_concurrent_bulk_requests", Runtime.getRuntime().availableProcessors() * 2);
    ByteSizeValue maxvolume = settings.getAsBytesSize("max_bulk_volume", ByteSizeValue.parseBytesSizeValue("10m", ""));
    TimeValue flushinterval = settings.getAsTime("flush_interval", TimeValue.timeValueSeconds(5));
    Settings.Builder clientSettings = Settings.settingsBuilder().put("cluster.name", settings.get("elasticsearch.cluster", "elasticsearch")).putArray("host", settings.getAsArray("elasticsearch.host")).put("port", settings.getAsInt("elasticsearch.port", 9300)).put("sniff", settings.getAsBoolean("elasticsearch.sniff", false)).put("autodiscover", settings.getAsBoolean("elasticsearch.autodiscover", false)).put("name", // prevents lookup of names.txt, we don't have it, and marks this node as "feeder". See also module load skipping in JDBCRiverPlugin
    "feeder").put("client.transport.ignore_cluster_name", // do not ignore cluster name setting
    false).put("client.transport.ping_timeout", // ping timeout
    settings.getAsTime("elasticsearch.timeout", TimeValue.timeValueSeconds(10))).put("client.transport.nodes_sampler_interval", // for sniff sampling
    settings.getAsTime("elasticsearch.timeout", TimeValue.timeValueSeconds(5))).put("path.plugins", // pointing to a non-exiting folder means, this disables loading site plugins
    ".dontexist").put("path.home", System.getProperty("path.home"));
    if (settings.get("transport.type") != null) {
        clientSettings.put("transport.type", settings.get("transport.type"));
    }
    // copy found.no transport settings
    Settings foundTransportSettings = settings.getAsSettings("transport.found");
    if (foundTransportSettings != null) {
        Map<String, String> foundTransportSettingsMap = foundTransportSettings.getAsMap();
        for (Map.Entry<String, String> entry : foundTransportSettingsMap.entrySet()) {
            clientSettings.put("transport.found." + entry.getKey(), entry.getValue());
        }
    }
    try {
        ClientAPI clientAPI = ClientBuilder.builder().put(settings).put(ClientBuilder.MAX_ACTIONS_PER_REQUEST, maxbulkactions).put(ClientBuilder.MAX_CONCURRENT_REQUESTS, maxconcurrentbulkrequests).put(ClientBuilder.MAX_VOLUME_PER_REQUEST, maxvolume).put(ClientBuilder.FLUSH_INTERVAL, flushinterval).setMetric(new ElasticsearchIngestMetric()).toBulkTransportClient();
    } catch (NoNodeAvailableException e) {
    // ok
    }
}