Examples with TimeValue org.graylog.shaded.elasticsearch7.org.elasticsearch.common.unit.TimeValue used on opensource projects

Example 66 with TimeValue

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.common.unit.TimeValue in project bw-calendar-engine by Bedework.

the class BwIndexEsImpl method fetchAllEntities.

private <T> List<T> fetchAllEntities(final String docType, final BuildEntity<T> be, final FilterBuilder filter) throws CalFacadeException {
    final SearchRequestBuilder srb = getClient().prepareSearch(targetIndex);
    srb.setTypes(docType);
    if (debug) {
        debug("fetchAllEntities: srb=" + srb);
    }
    int tries = 0;
    // int ourPos = 0;
    final int ourCount = maxFetchCount;
    final List<T> res = new ArrayList<>();
    SearchResponse scrollResp = srb.setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setPostFilter(filter).setSize(ourCount).execute().actionGet();
    if (scrollResp.status() != RestStatus.OK) {
        if (debug) {
            debug("Search returned status " + scrollResp.status());
        }
    }
    for (; ; ) {
        if (tries > absoluteMaxTries) {
            // huge count or we screwed up
            warn("Indexer: too many tries");
            break;
        }
        scrollResp = getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
        if (scrollResp.status() != RestStatus.OK) {
            if (debug) {
                debug("Search returned status " + scrollResp.status());
            }
        }
        final SearchHits hits = scrollResp.getHits();
        // Break condition: No hits are returned
        if (hits.hits().length == 0) {
            break;
        }
        for (final SearchHit hit : hits) {
            // Handle the hit...
            final T ent = be.make(getEntityBuilder(hit.sourceAsMap()));
            if (ent == null) {
                // No access
                continue;
            }
            res.add(ent);
        // ourPos++;
        }
        tries++;
    }
    return res;
}

Example 67 with TimeValue

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.common.unit.TimeValue in project core-ng-project by neowu.

the class ElasticSearchImpl method createClient.

protected Client createClient() {
    if (addresses.isEmpty())
        throw new Error("addresses must not be empty");
    StopWatch watch = new StopWatch();
    try {
        Settings.Builder settings = Settings.builder();
        settings.put(NetworkService.TCP_CONNECT_TIMEOUT.getKey(), new TimeValue(timeout.toMillis())).put(TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.getKey(), new TimeValue(timeout.toMillis())).put(TransportClient.CLIENT_TRANSPORT_PING_TIMEOUT.getKey(), new TimeValue(timeout.toMillis())).put(TransportClient.CLIENT_TRANSPORT_IGNORE_CLUSTER_NAME.getKey(), // refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/transport-client.html
        "true");
        if (sniff) {
            settings.put(TransportClient.CLIENT_TRANSPORT_SNIFF.getKey(), true);
        }
        TransportClient client = new PreBuiltTransportClient(settings.build());
        addresses.forEach(client::addTransportAddress);
        return client;
    } finally {
        logger.info("create elasticsearch client, addresses={}, elapsedTime={}", addresses, watch.elapsedTime());
    }
}

Example 68 with TimeValue

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.common.unit.TimeValue in project incubator-sdap-mudrod by apache.

the class SessionExtractor method bulidSessionDatasetRDD.

public JavaPairRDD<String, List<String>> bulidSessionDatasetRDD(Properties props, ESDriver es, SparkDriver spark) {
    List<String> result = new ArrayList<>();
    List<String> logIndexList = es.getIndexListWithPrefix(props.getProperty(MudrodConstants.LOG_INDEX));
    for (String logIndex : logIndexList) {
        SearchResponse scrollResp = es.getClient().prepareSearch(logIndex).setTypes(MudrodConstants.SESSION_STATS_TYPE).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.matchAllQuery()).setSize(100).execute().actionGet();
        while (true) {
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                Map<String, Object> session = hit.getSource();
                String sessionID = (String) session.get("SessionID");
                String views = (String) session.get("views");
                if (views != null && !"".equals(views)) {
                    String sessionItems = sessionID + ":" + views;
                    result.add(sessionItems);
                }
            }
            scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
    }
    JavaRDD<String> sessionRDD = spark.sc.parallelize(result);
    return sessionRDD.mapToPair(new PairFunction<String, String, List<String>>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Tuple2<String, List<String>> call(String sessionitem) throws Exception {
            String[] splits = sessionitem.split(":");
            String sessionId = splits[0];
            List<String> itemList = new ArrayList<>();
            String items = splits[1];
            String[] itemArr = items.split(",");
            for (String item : itemArr) {
                if (!itemList.contains(item))
                    itemList.add(item);
            }
            return new Tuple2<>(sessionId, itemList);
        }
    });
}

Example 69 with TimeValue

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.common.unit.TimeValue in project incubator-sdap-mudrod by apache.

the class NormalizeFeatures method normalizeMetadataVariables.

public void normalizeMetadataVariables(ESDriver es) {
    es.createBulkProcessor();
    SearchResponse scrollResp = es.getClient().prepareSearch(indexName).setTypes(metadataType).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.matchAllQuery()).setSize(100).execute().actionGet();
    while (true) {
        for (SearchHit hit : scrollResp.getHits().getHits()) {
            Map<String, Object> metadata = hit.getSource();
            Map<String, Object> updatedValues = new HashMap<>();
            // !!!important change to other normalizer class when using other metadata
            MetadataFeature normalizer = new PODAACMetadataFeature();
            normalizer.normalizeMetadataVariables(metadata, updatedValues);
            UpdateRequest ur = es.generateUpdateRequest(indexName, metadataType, hit.getId(), updatedValues);
            es.getBulkProcessor().add(ur);
        }
        scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
        if (scrollResp.getHits().getHits().length == 0) {
            break;
        }
    }
    es.destroyBulkProcessor();
}

Example 70 with TimeValue

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.common.unit.TimeValue in project incubator-sdap-mudrod by apache.

the class LinkageTriple method standardTriples.

public static void standardTriples(ESDriver es, String index, String type) throws IOException {
    es.createBulkProcessor();
    SearchResponse sr = es.getClient().prepareSearch(index).setTypes(type).setQuery(QueryBuilders.matchAllQuery()).setSize(0).addAggregation(AggregationBuilders.terms("concepts").field("concept_A").size(0)).execute().actionGet();
    Terms concepts = sr.getAggregations().get("concepts");
    for (Terms.Bucket entry : concepts.getBuckets()) {
        String concept = (String) entry.getKey();
        double maxSim = LinkageTriple.getMaxSimilarity(es, index, type, concept);
        if (maxSim == 1.0) {
            continue;
        }
        SearchResponse scrollResp = es.getClient().prepareSearch(index).setTypes(type).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.termQuery("concept_A", concept)).addSort("weight", SortOrder.DESC).setSize(100).execute().actionGet();
        while (true) {
            for (SearchHit hit : scrollResp.getHits().getHits()) {
                Map<String, Object> metadata = hit.getSource();
                double sim = (double) metadata.get("weight");
                double newSim = sim / maxSim;
                UpdateRequest ur = es.generateUpdateRequest(index, type, hit.getId(), "weight", Double.parseDouble(df.format(newSim)));
                es.getBulkProcessor().add(ur);
            }
            scrollResp = es.getClient().prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
            if (scrollResp.getHits().getHits().length == 0) {
                break;
            }
        }
    }
    es.destroyBulkProcessor();
}