Examples with SearchHit org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit used on opensource projects

Example 16 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project elasticsearch by elastic.

the class ExceptionRetryIT method testRetryDueToExceptionOnNetworkLayer.

/**
     * Tests retry mechanism when indexing. If an exception occurs when indexing then the indexing request is tried again before finally
     * failing. If auto generated ids are used this must not lead to duplicate ids
     * see https://github.com/elastic/elasticsearch/issues/8788
     */
public void testRetryDueToExceptionOnNetworkLayer() throws ExecutionException, InterruptedException, IOException {
    final AtomicBoolean exceptionThrown = new AtomicBoolean(false);
    int numDocs = scaledRandomIntBetween(100, 1000);
    Client client = internalCluster().coordOnlyNodeClient();
    NodesStatsResponse nodeStats = client().admin().cluster().prepareNodesStats().get();
    NodeStats unluckyNode = randomFrom(nodeStats.getNodes().stream().filter((s) -> s.getNode().isDataNode()).collect(Collectors.toList()));
    assertAcked(client().admin().indices().prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 1).put("index.number_of_shards", 5)));
    ensureGreen("index");
    logger.info("unlucky node: {}", unluckyNode.getNode());
    //create a transport service that throws a ConnectTransportException for one bulk request and therefore triggers a retry.
    for (NodeStats dataNode : nodeStats.getNodes()) {
        MockTransportService mockTransportService = ((MockTransportService) internalCluster().getInstance(TransportService.class, dataNode.getNode().getName()));
        mockTransportService.addDelegate(internalCluster().getInstance(TransportService.class, unluckyNode.getNode().getName()), new MockTransportService.DelegateTransport(mockTransportService.original()) {

            @Override
            protected void sendRequest(Connection connection, long requestId, String action, TransportRequest request, TransportRequestOptions options) throws IOException {
                super.sendRequest(connection, requestId, action, request, options);
                if (action.equals(TransportShardBulkAction.ACTION_NAME) && exceptionThrown.compareAndSet(false, true)) {
                    logger.debug("Throw ConnectTransportException");
                    throw new ConnectTransportException(connection.getNode(), action);
                }
            }
        });
    }
    BulkRequestBuilder bulkBuilder = client.prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        XContentBuilder doc = null;
        doc = jsonBuilder().startObject().field("foo", "bar").endObject();
        bulkBuilder.add(client.prepareIndex("index", "type").setSource(doc));
    }
    BulkResponse response = bulkBuilder.get();
    if (response.hasFailures()) {
        for (BulkItemResponse singleIndexRespons : response.getItems()) {
            if (singleIndexRespons.isFailed()) {
                fail("None of the bulk items should fail but got " + singleIndexRespons.getFailureMessage());
            }
        }
    }
    refresh();
    SearchResponse searchResponse = client().prepareSearch("index").setSize(numDocs * 2).addStoredField("_id").get();
    Set<String> uniqueIds = new HashSet();
    long dupCounter = 0;
    boolean found_duplicate_already = false;
    for (int i = 0; i < searchResponse.getHits().getHits().length; i++) {
        if (!uniqueIds.add(searchResponse.getHits().getHits()[i].getId())) {
            if (!found_duplicate_already) {
                SearchResponse dupIdResponse = client().prepareSearch("index").setQuery(termQuery("_id", searchResponse.getHits().getHits()[i].getId())).setExplain(true).get();
                assertThat(dupIdResponse.getHits().getTotalHits(), greaterThan(1L));
                logger.info("found a duplicate id:");
                for (SearchHit hit : dupIdResponse.getHits()) {
                    logger.info("Doc {} was found on shard {}", hit.getId(), hit.getShard().getShardId());
                }
                logger.info("will not print anymore in case more duplicates are found.");
                found_duplicate_already = true;
            }
            dupCounter++;
        }
    }
    assertSearchResponse(searchResponse);
    assertThat(dupCounter, equalTo(0L));
    assertHitCount(searchResponse, numDocs);
    IndicesStatsResponse index = client().admin().indices().prepareStats("index").clear().setSegments(true).get();
    IndexStats indexStats = index.getIndex("index");
    long maxUnsafeAutoIdTimestamp = Long.MIN_VALUE;
    for (IndexShardStats indexShardStats : indexStats) {
        for (ShardStats shardStats : indexShardStats) {
            SegmentsStats segments = shardStats.getStats().getSegments();
            maxUnsafeAutoIdTimestamp = Math.max(maxUnsafeAutoIdTimestamp, segments.getMaxUnsafeAutoIdTimestamp());
        }
    }
    assertTrue("exception must have been thrown otherwise setup is broken", exceptionThrown.get());
    assertTrue("maxUnsafeAutoIdTimestamp must be > than 0 we have at least one retry", maxUnsafeAutoIdTimestamp > -1);
}

Example 17 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project elasticsearch by elastic.

the class SimpleIndexTemplateIT method testIndexTemplateWithAliases.

public void testIndexTemplateWithAliases() throws Exception {
    client().admin().indices().preparePutTemplate("template_with_aliases").setPatterns(Collections.singletonList("te*")).addMapping("type1", "{\"type1\" : {\"properties\" : {\"value\" : {\"type\" : \"text\"}}}}", XContentType.JSON).addAlias(new Alias("simple_alias")).addAlias(new Alias("templated_alias-{index}")).addAlias(new Alias("filtered_alias").filter("{\"type\":{\"value\":\"type2\"}}")).addAlias(new Alias("complex_filtered_alias").filter(QueryBuilders.termsQuery("_type", "typeX", "typeY", "typeZ"))).get();
    assertAcked(prepareCreate("test_index").addMapping("type1").addMapping("type2").addMapping("typeX").addMapping("typeY").addMapping("typeZ"));
    ensureGreen();
    client().prepareIndex("test_index", "type1", "1").setSource("field", "A value").get();
    client().prepareIndex("test_index", "type2", "2").setSource("field", "B value").get();
    client().prepareIndex("test_index", "typeX", "3").setSource("field", "C value").get();
    client().prepareIndex("test_index", "typeY", "4").setSource("field", "D value").get();
    client().prepareIndex("test_index", "typeZ", "5").setSource("field", "E value").get();
    GetAliasesResponse getAliasesResponse = client().admin().indices().prepareGetAliases().setIndices("test_index").get();
    assertThat(getAliasesResponse.getAliases().size(), equalTo(1));
    assertThat(getAliasesResponse.getAliases().get("test_index").size(), equalTo(4));
    refresh();
    SearchResponse searchResponse = client().prepareSearch("test_index").get();
    assertHitCount(searchResponse, 5L);
    searchResponse = client().prepareSearch("simple_alias").get();
    assertHitCount(searchResponse, 5L);
    searchResponse = client().prepareSearch("templated_alias-test_index").get();
    assertHitCount(searchResponse, 5L);
    searchResponse = client().prepareSearch("filtered_alias").get();
    assertHitCount(searchResponse, 1L);
    assertThat(searchResponse.getHits().getAt(0).getType(), equalTo("type2"));
    // Search the complex filter alias
    searchResponse = client().prepareSearch("complex_filtered_alias").get();
    assertHitCount(searchResponse, 3L);
    Set<String> types = new HashSet<>();
    for (SearchHit searchHit : searchResponse.getHits().getHits()) {
        types.add(searchHit.getType());
    }
    assertThat(types.size(), equalTo(3));
    assertThat(types, containsInAnyOrder("typeX", "typeY", "typeZ"));
}

Example 18 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project elasticsearch by elastic.

the class SearchPhaseController method getHits.

private SearchHits getHits(ReducedQueryPhase reducedQueryPhase, boolean ignoreFrom, ScoreDoc[] sortedDocs, AtomicArray<? extends QuerySearchResultProvider> fetchResultsArr) {
    List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> fetchResults = fetchResultsArr.asList();
    boolean sorted = false;
    int sortScoreIndex = -1;
    if (reducedQueryPhase.oneResult.topDocs() instanceof TopFieldDocs) {
        TopFieldDocs fieldDocs = (TopFieldDocs) reducedQueryPhase.oneResult.queryResult().topDocs();
        if (fieldDocs instanceof CollapseTopFieldDocs && fieldDocs.fields.length == 1 && fieldDocs.fields[0].getType() == SortField.Type.SCORE) {
            sorted = false;
        } else {
            sorted = true;
            for (int i = 0; i < fieldDocs.fields.length; i++) {
                if (fieldDocs.fields[i].getType() == SortField.Type.SCORE) {
                    sortScoreIndex = i;
                }
            }
        }
    }
    // clean the fetch counter
    for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : fetchResults) {
        entry.value.fetchResult().initCounter();
    }
    int from = ignoreFrom ? 0 : reducedQueryPhase.oneResult.queryResult().from();
    int numSearchHits = (int) Math.min(reducedQueryPhase.fetchHits - from, reducedQueryPhase.oneResult.size());
    // with collapsing we can have more fetch hits than sorted docs
    numSearchHits = Math.min(sortedDocs.length, numSearchHits);
    // merge hits
    List<SearchHit> hits = new ArrayList<>();
    if (!fetchResults.isEmpty()) {
        for (int i = 0; i < numSearchHits; i++) {
            ScoreDoc shardDoc = sortedDocs[i];
            QuerySearchResultProvider fetchResultProvider = fetchResultsArr.get(shardDoc.shardIndex);
            if (fetchResultProvider == null) {
                continue;
            }
            FetchSearchResult fetchResult = fetchResultProvider.fetchResult();
            int index = fetchResult.counterGetAndIncrement();
            if (index < fetchResult.hits().internalHits().length) {
                SearchHit searchHit = fetchResult.hits().internalHits()[index];
                searchHit.score(shardDoc.score);
                searchHit.shard(fetchResult.shardTarget());
                if (sorted) {
                    FieldDoc fieldDoc = (FieldDoc) shardDoc;
                    searchHit.sortValues(fieldDoc.fields, reducedQueryPhase.oneResult.sortValueFormats());
                    if (sortScoreIndex != -1) {
                        searchHit.score(((Number) fieldDoc.fields[sortScoreIndex]).floatValue());
                    }
                }
                hits.add(searchHit);
            }
        }
    }
    return new SearchHits(hits.toArray(new SearchHit[hits.size()]), reducedQueryPhase.totalHits, reducedQueryPhase.maxScore);
}

Example 19 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project sonarqube by SonarSource.

the class UserUpdaterTest method update_user.

@Test
public void update_user() {
    UserDto user = db.users().insertUser(newLocalUser(DEFAULT_LOGIN, "Marius", "marius@email.com").setScmAccounts(asList("ma", "marius33")).setSalt("79bd6a8e79fb8c76ac8b121cc7e8e11ad1af8365").setCryptedPassword("650d2261c98361e2f67f90ce5c65a95e7d8ea2fg").setCreatedAt(PAST).setUpdatedAt(PAST));
    createDefaultGroup();
    underTest.update(session, UpdateUser.create(DEFAULT_LOGIN).setName("Marius2").setEmail("marius2@mail.com").setPassword("password2").setScmAccounts(newArrayList("ma2")));
    session.commit();
    UserDto updatedUser = dbClient.userDao().selectByLogin(session, DEFAULT_LOGIN);
    assertThat(updatedUser.isActive()).isTrue();
    assertThat(updatedUser.getName()).isEqualTo("Marius2");
    assertThat(updatedUser.getEmail()).isEqualTo("marius2@mail.com");
    assertThat(updatedUser.getScmAccountsAsList()).containsOnly("ma2");
    assertThat(updatedUser.getSalt()).isNotEqualTo(user.getSalt());
    assertThat(updatedUser.getCryptedPassword()).isNotEqualTo(user.getCryptedPassword());
    assertThat(updatedUser.getCreatedAt()).isEqualTo(PAST);
    assertThat(updatedUser.getUpdatedAt()).isEqualTo(NOW);
    List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER);
    assertThat(indexUsers).hasSize(1);
    assertThat(indexUsers.get(0).getSource()).contains(entry("login", DEFAULT_LOGIN), entry("name", "Marius2"), entry("email", "marius2@mail.com"));
}

Example 20 with SearchHit

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.SearchHit in project sonarqube by SonarSource.

the class UserUpdaterTest method create_user.

@Test
public void create_user() {
    createDefaultGroup();
    UserDto dto = underTest.create(NewUser.builder().setLogin("user").setName("User").setEmail("user@mail.com").setPassword("PASSWORD").setScmAccounts(ImmutableList.of("u1", "u_1", "User 1")).build());
    assertThat(dto.getId()).isNotNull();
    assertThat(dto.getLogin()).isEqualTo("user");
    assertThat(dto.getName()).isEqualTo("User");
    assertThat(dto.getEmail()).isEqualTo("user@mail.com");
    assertThat(dto.getScmAccountsAsList()).containsOnly("u1", "u_1", "User 1");
    assertThat(dto.isActive()).isTrue();
    assertThat(dto.isLocal()).isTrue();
    assertThat(dto.getSalt()).isNotNull();
    assertThat(dto.getCryptedPassword()).isNotNull();
    assertThat(dto.getCreatedAt()).isEqualTo(1418215735482L);
    assertThat(dto.getUpdatedAt()).isEqualTo(1418215735482L);
    assertThat(dbClient.userDao().selectByLogin(session, "user").getId()).isEqualTo(dto.getId());
    List<SearchHit> indexUsers = es.getDocuments(UserIndexDefinition.INDEX_TYPE_USER);
    assertThat(indexUsers).hasSize(1);
    assertThat(indexUsers.get(0).getSource()).contains(entry("login", "user"), entry("name", "User"), entry("email", "user@mail.com"));
}