Examples with OneToManyResilientIndexingListener org.sonar.server.es.OneToManyResilientIndexingListener used on opensource projects

Example 1 with OneToManyResilientIndexingListener

use of org.sonar.server.es.OneToManyResilientIndexingListener in project sonarqube by SonarSource.

the class ComponentIndexer method index.

@Override
public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) {
    if (items.isEmpty()) {
        return new IndexingResult();
    }
    OneToManyResilientIndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items);
    BulkIndexer bulkIndexer = new BulkIndexer(esClient, TYPE_COMPONENT, Size.REGULAR, listener);
    bulkIndexer.start();
    Set<String> branchUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet(items.size()));
    Set<String> remaining = new HashSet<>(branchUuids);
    for (String branchUuid : branchUuids) {
        // TODO allow scrolling multiple projects at the same time
        dbClient.componentDao().scrollForIndexing(dbSession, branchUuid, context -> {
            ComponentDto dto = context.getResultObject();
            bulkIndexer.add(toDocument(dto).toIndexRequest());
            remaining.remove(dto.projectUuid());
        });
    }
    // the remaining uuids reference projects that don't exist in db. They must
    // be deleted from index.
    remaining.forEach(projectUuid -> addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid));
    return bulkIndexer.stop();
}

Example 2 with OneToManyResilientIndexingListener

use of org.sonar.server.es.OneToManyResilientIndexingListener in project sonarqube by SonarSource.

the class IssueIndexer method doIndexProjectItems.

private IndexingResult doIndexProjectItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByProjectUuid) {
    if (itemsByProjectUuid.isEmpty()) {
        return new IndexingResult();
    }
    // one project, referenced by es_queue.doc_id = many issues
    IndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, itemsByProjectUuid.values());
    BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener);
    bulkIndexer.start();
    for (String projectUuid : itemsByProjectUuid.keySet()) {
        // TODO support loading of multiple projects in a single SQL request
        try (IssueIterator issues = issueIteratorFactory.createForProject(projectUuid)) {
            if (issues.hasNext()) {
                do {
                    IssueDoc doc = issues.next();
                    bulkIndexer.add(newIndexRequest(doc));
                } while (issues.hasNext());
            } else {
                // project does not exist or has no issues. In both case
                // all the documents related to this project are deleted.
                addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid);
            }
        }
    }
    return bulkIndexer.stop();
}