Examples with EsQueueDto org.sonar.db.es.EsQueueDto used on opensource projects

Example 1 with EsQueueDto

use of org.sonar.db.es.EsQueueDto in project sonarqube by SonarSource.

the class RecoveryIndexer method recover.

@VisibleForTesting
void recover() {
    try (DbSession dbSession = dbClient.openSession(false)) {
        Profiler profiler = Profiler.create(LOGGER).start();
        long beforeDate = system2.now() - minAgeInMs;
        IndexingResult result = new IndexingResult();
        Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
        while (!items.isEmpty()) {
            IndexingResult loopResult = new IndexingResult();
            groupItemsByDocType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems)));
            result.add(loopResult);
            if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) {
                LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal());
                break;
            }
            if (loopResult.getTotal() == 0L) {
                break;
            }
            items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit);
        }
        if (result.getTotal() > 0L) {
            profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures()));
        }
    } catch (Throwable t) {
        LOGGER.error(LOG_PREFIX + "fail to recover documents", t);
    }
}

Example 2 with EsQueueDto

use of org.sonar.db.es.EsQueueDto in project sonarqube by SonarSource.

the class PermissionIndexerTest method deletion_resilience_will_deindex_projects.

@Test
public void deletion_resilience_will_deindex_projects() {
    ComponentDto project1 = createUnindexedPublicProject();
    ComponentDto project2 = createUnindexedPublicProject();
    // UserDto user1 = db.users().insertUser();
    indexOnStartup();
    assertThat(es.countDocuments(INDEX_TYPE_FOO_AUTH)).isEqualTo(2);
    // Simulate a indexation issue
    db.getDbClient().purgeDao().deleteProject(db.getSession(), project1.uuid(), PROJECT, project1.name(), project1.getKey());
    underTest.prepareForRecovery(db.getSession(), asList(project1.uuid()), ProjectIndexer.Cause.PROJECT_DELETION);
    assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isOne();
    Collection<EsQueueDto> esQueueDtos = db.getDbClient().esQueueDao().selectForRecovery(db.getSession(), Long.MAX_VALUE, 2);
    underTest.index(db.getSession(), esQueueDtos);
    assertThat(db.countRowsOfTable(db.getSession(), "es_queue")).isZero();
    assertThat(es.countDocuments(INDEX_TYPE_FOO_AUTH)).isOne();
}

Example 3 with EsQueueDto

use of org.sonar.db.es.EsQueueDto in project sonarqube by SonarSource.

the class IssueIndexer method index.

@Override
public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
    ListMultimap<String, EsQueueDto> itemsByIssueKey = ArrayListMultimap.create();
    ListMultimap<String, EsQueueDto> itemsByProjectKey = ArrayListMultimap.create();
    items.forEach(i -> {
        if (ID_TYPE_ISSUE_KEY.equals(i.getDocIdType())) {
            itemsByIssueKey.put(i.getDocId(), i);
        } else if (ID_TYPE_PROJECT_UUID.equals(i.getDocIdType())) {
            itemsByProjectKey.put(i.getDocId(), i);
        } else {
            LOGGER.error("Unsupported es_queue.doc_id_type for issues. Manual fix is required: " + i);
        }
    });
    IndexingResult result = new IndexingResult();
    result.add(doIndexIssueItems(dbSession, itemsByIssueKey));
    result.add(doIndexProjectItems(dbSession, itemsByProjectKey));
    return result;
}

Example 4 with EsQueueDto

use of org.sonar.db.es.EsQueueDto in project sonarqube by SonarSource.

the class OneToOneResilientIndexingListenerTest method onSuccess_deletes_all_the_rows_with_same_doc_id.

/**
 * ES_QUEUE can contain multiple times the same document, for instance
 * when an issue has been updated multiple times in a row without
 * being successfully indexed.
 * Elasticsearch response does not make difference between the different
 * occurrences (and nevertheless it would be useless). So all the
 * occurrences are marked as successfully indexed if a single request
 * passes.
 */
@Test
public void onSuccess_deletes_all_the_rows_with_same_doc_id() {
    EsQueueDto item1 = insertInQueue(TYPE_ISSUE, "foo");
    // same id as item1
    EsQueueDto item2 = insertInQueue(TYPE_ISSUE, item1.getDocId());
    EsQueueDto item3 = insertInQueue(TYPE_ISSUE, "bar");
    db.commit();
    IndexingListener underTest = newListener(asList(item1, item2, item3));
    underTest.onSuccess(asList(toDocId(item1)));
    assertThatEsTableContainsOnly(item3);
}

Example 5 with EsQueueDto

use of org.sonar.db.es.EsQueueDto in project sonarqube by SonarSource.

the class ProjectIndexersTest method commitAndIndexByProjectUuids_calls_indexer_with_only_its_supported_items.

@Test
public void commitAndIndexByProjectUuids_calls_indexer_with_only_its_supported_items() {
    EsQueueDto item1a = EsQueueDto.create("fake/fake1", "P1");
    EsQueueDto item1b = EsQueueDto.create("fake/fake1", "P1");
    EsQueueDto item2 = EsQueueDto.create("fake/fake2", "P1");
    FakeIndexer indexer1 = new FakeIndexer(asList(item1a, item1b));
    FakeIndexer indexer2 = new FakeIndexer(singletonList(item2));
    DbSession dbSession = mock(DbSession.class);
    ProjectIndexersImpl underTest = new ProjectIndexersImpl(indexer1, indexer2);
    underTest.commitAndIndexByProjectUuids(dbSession, singletonList("P1"), ProjectIndexer.Cause.PROJECT_CREATION);
    assertThat(indexer1.calledItems).containsExactlyInAnyOrder(item1a, item1b);
    assertThat(indexer2.calledItems).containsExactlyInAnyOrder(item2);
}