Examples with ElasticsearchIndexer org.icij.datashare.text.indexing.elasticsearch.ElasticsearchIndexer used on opensource projects

Example 1 with ElasticsearchIndexer

use of org.icij.datashare.text.indexing.elasticsearch.ElasticsearchIndexer in project datashare by ICIJ.

the class DocumentIngestor method main.

public static void main(String[] args) {
    OptionSet optionSet = parseArgs(args);
    Integer nbThreads = (Integer) optionSet.valueOf("t");
    Integer bulkSize = (Integer) optionSet.valueOf("bulkSize");
    String elasticsearchUrl = (String) optionSet.valueOf("elasticsearchAddress");
    String indexName = (String) optionSet.valueOf("indexName");
    Integer nbDocuments = (Integer) optionSet.valueOf("nbDocuments");
    PropertiesProvider propertiesProvider = new PropertiesProvider(new HashMap<String, String>() {

        {
            put(INDEX_ADDRESS_PROP, elasticsearchUrl);
        }
    });
    Indexer indexer = new ElasticsearchIndexer(ElasticsearchConfiguration.createESClient(propertiesProvider), propertiesProvider);
    logger.info("ingest {} documents in elasticsearch {} with bulk of {} and {} threads", nbDocuments, elasticsearchUrl, bulkSize, nbThreads);
    executorService = Executors.newFixedThreadPool(nbThreads);
    new DocumentProducer(nbDocuments, bulkSize).run();
    IntStream.range(0, nbThreads).forEach(n -> executorService.submit(new DocumentConsumer(indexer, indexName)));
    executorService.shutdown();
}