Examples with DocumentMapping com.b2international.index.mapping.DocumentMapping used on opensource projects

Example 1 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsDocumentSearcher method aggregate.

@Override
public <T> Aggregation<T> aggregate(AggregationBuilder<T> aggregation) throws IOException {
    final String aggregationName = aggregation.getName();
    final EsClient client = admin.client();
    final DocumentMapping mapping = admin.mappings().getMapping(aggregation.getFrom());
    final EsQueryBuilder esQueryBuilder = new EsQueryBuilder(mapping, admin.settings(), admin.log());
    final QueryBuilder esQuery = esQueryBuilder.build(aggregation.getQuery());
    final SearchRequest req = new SearchRequest(admin.getTypeIndex(mapping));
    final SearchSourceBuilder reqSource = req.source().query(esQuery).size(0).trackScores(false).trackTotalHitsUpTo(Integer.MAX_VALUE);
    // field selection
    final boolean fetchSource = applySourceFiltering(aggregation.getFields(), mapping, reqSource);
    reqSource.aggregation(toEsAggregation(mapping, aggregation, fetchSource));
    SearchResponse response = null;
    try {
        response = client.search(req);
    } catch (Exception e) {
        admin.log().error("Couldn't execute aggregation", e);
        throw new IndexException("Couldn't execute aggregation: " + e.getMessage(), null);
    }
    ImmutableMap.Builder<Object, Bucket<T>> buckets = ImmutableMap.builder();
    Aggregations topLevelAggregations = response.getAggregations();
    Nested nested = topLevelAggregations.get(nestedAggName(aggregation));
    Terms aggregationResult;
    if (nested != null) {
        aggregationResult = nested.getAggregations().get(aggregationName);
    } else {
        aggregationResult = topLevelAggregations.get(aggregationName);
    }
    for (org.elasticsearch.search.aggregations.bucket.terms.Terms.Bucket bucket : aggregationResult.getBuckets()) {
        final TopHits topHits;
        if (nested != null) {
            final ReverseNested reverseNested = bucket.getAggregations().get(reverseNestedAggName(aggregation));
            topHits = reverseNested.getAggregations().get(topHitsAggName(aggregation));
        } else {
            topHits = bucket.getAggregations().get(topHitsAggName(aggregation));
        }
        Hits<T> hits;
        if (topHits != null) {
            hits = toHits(aggregation.getSelect(), List.of(aggregation.getFrom()), aggregation.getFields(), fetchSource, aggregation.getBucketHitsLimit(), (int) bucket.getDocCount(), null, topHits.getHits());
        } else {
            hits = new Hits<>(Collections.emptyList(), null, aggregation.getBucketHitsLimit(), (int) bucket.getDocCount());
        }
        buckets.put(bucket.getKey(), new Bucket<>(bucket.getKey(), hits));
    }
    return new Aggregation<>(aggregationName, buckets.build());
}

Example 2 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsDocumentSearcher method search.

@Override
public <T> Hits<T> search(Query<T> query) throws IOException {
    Stopwatch w = Stopwatch.createStarted();
    admin.log().trace("Executing query '{}'", query);
    final EsClient client = admin.client();
    final List<DocumentMapping> mappings = admin.mappings().getDocumentMapping(query);
    final DocumentMapping primaryMapping = Iterables.getFirst(mappings, null);
    // Restrict variables to the theoretical maximum
    final int limit = query.getLimit();
    final int toRead = Ints.min(limit, resultWindow);
    // TODO support multiple document mappings during query building
    final EsQueryBuilder esQueryBuilder = new EsQueryBuilder(primaryMapping, admin.settings(), admin.log());
    final QueryBuilder esQuery = esQueryBuilder.build(query.getWhere());
    final SearchRequest req = new SearchRequest(admin.getTypeIndexes(mappings).toArray(length -> new String[length]));
    // configure caching
    req.requestCache(query.isCached());
    final SearchSourceBuilder reqSource = req.source().size(toRead).query(esQuery).trackScores(esQueryBuilder.needsScoring()).trackTotalHitsUpTo(Integer.MAX_VALUE);
    // field selection
    final boolean fetchSource = applySourceFiltering(query.getFields(), primaryMapping, reqSource);
    // ES internals require loading the _id field when we require the _source
    if (fetchSource) {
        reqSource.storedFields(STORED_FIELDS_ID_ONLY);
    } else {
        reqSource.storedFields(STORED_FIELDS_NONE);
    }
    // paging config
    final boolean isLocalStreaming = limit > resultWindow;
    final boolean isLiveStreaming = !Strings.isNullOrEmpty(query.getSearchAfter());
    if (isLocalStreaming) {
        checkArgument(!isLiveStreaming, "Cannot use searchAfter when requesting more items (%s) than the configured result window (%s).", limit, resultWindow);
    } else if (isLiveStreaming) {
        reqSource.searchAfter(fromSearchAfterToken(query.getSearchAfter()));
    }
    // sorting config with a default sort field based on scroll config
    addSort(primaryMapping, reqSource, query.getSortBy());
    // disable explain explicitly, just in case
    reqSource.explain(false);
    // disable version field explicitly, just in case
    reqSource.version(false);
    // perform search
    SearchResponse response = null;
    try {
        response = client.search(req);
    } catch (Exception e) {
        if (e instanceof ElasticsearchStatusException && ((ElasticsearchStatusException) e).status() == RestStatus.BAD_REQUEST) {
            throw new IllegalArgumentException(e.getMessage(), e);
        }
        admin.log().error("Couldn't execute query", e);
        throw new IndexException("Couldn't execute query: " + e.getMessage(), null);
    }
    SearchHits responseHits = response.getHits();
    final TotalHits total = responseHits.getTotalHits();
    checkState(total.relation == Relation.EQUAL_TO, "Searches should always track total hits accurately");
    final int totalHitCount = (int) total.value;
    final SearchHit[] firstHits = responseHits.getHits();
    final int firstCount = firstHits.length;
    final int remainingCount = Math.min(limit, totalHitCount) - firstCount;
    // Add the first set of results
    final ImmutableList.Builder<SearchHit> allHits = ImmutableList.builder();
    allHits.addAll(responseHits);
    // If the client requested all data at once and there are more hits to retrieve, collect them all as part of the request
    if (isLocalStreaming && remainingCount > 0) {
        admin.log().warn("Returning all matches (totalHits: '{}') larger than the currently configured result_window ('{}') might not be the most " + "efficient way of getting the data. Consider using the index pagination API (searchAfter) instead.", totalHitCount, resultWindow);
        while (true) {
            // Extract searchAfter values for the next set of results
            final SearchHit lastHit = Iterables.getLast(responseHits, null);
            if (lastHit == null) {
                break;
            }
            reqSource.searchAfter(lastHit.getSortValues());
            // Request more search results, adding them to the list builder
            response = client.search(req);
            responseHits = response.getHits();
            allHits.addAll(responseHits);
        }
    }
    final Class<T> select = query.getSelection().getSelect();
    final List<Class<?>> from = query.getSelection().getFrom();
    final Hits<T> hits = toHits(select, from, query.getFields(), fetchSource, limit, totalHitCount, query.getSortBy(), allHits.build());
    admin.log().trace("Executed query '{}' in '{}'", query, w);
    return hits;
}

Example 3 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsDocumentSearcher method get.

@Override
public <T> Iterable<T> get(Class<T> type, Iterable<String> keys) throws IOException {
    final DocumentMapping mapping = admin.mappings().getMapping(type);
    List<String> allKeys = ImmutableList.copyOf(keys);
    if (allKeys.size() > maxTermsCount) {
        List<T> results = Lists.newArrayListWithExpectedSize(allKeys.size());
        for (List<String> currentKeys : Lists.partition(allKeys, maxTermsCount)) {
            results.addAll(search(Query.select(type).where(Expressions.matchAny(mapping.getIdField(), currentKeys)).limit(currentKeys.size()).build()).getHits());
        }
        return results;
    } else {
        return search(Query.select(type).where(Expressions.matchAny(mapping.getIdField(), allKeys)).limit(allKeys.size()).build()).getHits();
    }
}

Example 4 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class StagingArea method collectConflicts.

private void collectConflicts(RevisionBranchChangeSet fromChangeSet, List<RevisionCompareDetail> fromChangeDetails, RevisionBranchChangeSet toChangeSet, List<RevisionCompareDetail> toChangeDetails, List<Conflict> conflictsToReport, Map<Class<? extends Revision>, Multimap<String, RevisionPropertyDiff>> propertyUpdatesToApply, RevisionConflictProcessor conflictProcessor) {
    List<Conflict> conflicts = newArrayList();
    for (Class<? extends Revision> type : ImmutableSet.copyOf(Iterables.concat(fromChangeSet.getAddedTypes(), toChangeSet.getAddedTypes()))) {
        final Set<String> newRevisionIdsOnSource = fromChangeSet.getAddedIds(type);
        final Set<String> newRevisionIdsOnTarget = toChangeSet.getAddedIds(type);
        final Set<String> addedInSourceAndTarget = Sets.intersection(newRevisionIdsOnSource, newRevisionIdsOnTarget);
        // check for added in both source and target conflicts
        if (!addedInSourceAndTarget.isEmpty()) {
            addedInSourceAndTarget.forEach(revisionId -> {
                conflicts.add(new AddedInSourceAndTargetConflict(ObjectId.of(type, revisionId)));
            });
        }
        // check deleted containers on target and report them as conflicts
        newRevisionIdsOnSource.forEach(newRevisionOnSource -> {
            ObjectId newRevisionOnSourceId = ObjectId.of(type, newRevisionOnSource);
            ObjectId requiredContainer = fromChangeSet.getContainerId(newRevisionOnSourceId);
            if (requiredContainer != null && toChangeSet.isRemoved(requiredContainer)) {
                conflicts.add(new AddedInSourceAndDetachedInTargetConflict(newRevisionOnSourceId, requiredContainer));
            }
        });
        // check deleted containers on source and report them as conflicts
        newRevisionIdsOnTarget.forEach(newRevisionOnTarget -> {
            ObjectId newRevisionOnTargetId = ObjectId.of(type, newRevisionOnTarget);
            ObjectId requiredContainer = toChangeSet.getContainerId(newRevisionOnTargetId);
            if (requiredContainer != null && fromChangeSet.isRemoved(requiredContainer)) {
                conflicts.add(new AddedInTargetAndDetachedInSourceConflict(requiredContainer, newRevisionOnTargetId));
            }
        });
    }
    // check property conflicts
    Set<String> changedRevisionIdsToCheck = newHashSet(toChangeSet.getChangedIds());
    Set<String> removedRevisionIdsToCheck = newHashSet(toChangeSet.getRemovedIds());
    for (Class<? extends Revision> type : fromChangeSet.getChangedTypes()) {
        final DocumentMapping mapping = index.admin().mappings().getMapping(type);
        final String docType = mapping.typeAsString();
        Set<String> changedRevisionIdsToMerge = newHashSet(fromChangeSet.getChangedIds(type));
        // first handle changed vs. removed
        Set<String> changedInSourceDetachedInTargetIds = Sets.newHashSet(Sets.intersection(changedRevisionIdsToMerge, removedRevisionIdsToCheck));
        if (!changedInSourceDetachedInTargetIds.isEmpty()) {
            // report any conflicts
            changedInSourceDetachedInTargetIds.forEach(changedInSourceDetachedInTargetId -> {
                List<RevisionPropertyDiff> sourceChanges = fromChangeDetails.stream().filter(detail -> detail.getObject().id().equals(changedInSourceDetachedInTargetId)).filter(detail -> !detail.isComponentChange()).map(change -> new RevisionPropertyDiff(change.getProperty(), change.getFromValue(), change.getValue())).collect(Collectors.toList());
                Conflict conflict = conflictProcessor.handleChangedInSourceDetachedInTarget(ObjectId.of(docType, changedInSourceDetachedInTargetId), sourceChanges);
                if (conflict != null) {
                    conflicts.add(conflict);
                }
            });
            // register them as revised on source from the target branch point of view
            revisionsToReviseOnMergeSource.putAll(type, changedInSourceDetachedInTargetIds);
            changedInSourceDetachedInTargetIds.forEach(id -> fromChangeSet.removeChanged(type, id));
            changedRevisionIdsToMerge.removeAll(changedInSourceDetachedInTargetIds);
        }
        // then handle changed vs. changed with the conflict processor
        Set<String> changedInSourceAndTargetIds = Sets.intersection(changedRevisionIdsToMerge, changedRevisionIdsToCheck);
        if (!changedInSourceAndTargetIds.isEmpty()) {
            final Map<String, Map<String, RevisionCompareDetail>> sourcePropertyChangesByObject = indexPropertyChangesByObject(fromChangeDetails);
            final Map<String, Map<String, RevisionCompareDetail>> targetPropertyChangesByObject = indexPropertyChangesByObject(toChangeDetails);
            for (String changedInSourceAndTargetId : changedInSourceAndTargetIds) {
                // take the prop changes from both paths
                final Map<String, RevisionCompareDetail> sourcePropertyChanges = sourcePropertyChangesByObject.remove(changedInSourceAndTargetId);
                final Map<String, RevisionCompareDetail> targetPropertyChanges = targetPropertyChangesByObject.remove(changedInSourceAndTargetId);
                if (sourcePropertyChanges != null) {
                    for (Entry<String, RevisionCompareDetail> sourceChange : sourcePropertyChanges.entrySet()) {
                        final String changedProperty = sourceChange.getKey();
                        final RevisionCompareDetail sourcePropertyChange = sourceChange.getValue();
                        final RevisionPropertyDiff sourceChangeDiff = new RevisionPropertyDiff(changedProperty, sourcePropertyChange.getFromValue(), sourcePropertyChange.getValue());
                        final RevisionCompareDetail targetPropertyChange = targetPropertyChanges == null ? null : targetPropertyChanges.get(changedProperty);
                        if (targetPropertyChange == null) {
                            // this property did not change in target, just apply directly on the target object via
                            if (!propertyUpdatesToApply.containsKey(type)) {
                                propertyUpdatesToApply.put(type, HashMultimap.create());
                            }
                            propertyUpdatesToApply.get(type).put(changedInSourceAndTargetId, sourceChangeDiff);
                            fromChangeSet.removeChanged(type, changedInSourceAndTargetId);
                        } else {
                            RevisionPropertyDiff targetChangeDiff = new RevisionPropertyDiff(targetPropertyChange.getProperty(), targetPropertyChange.getFromValue(), targetPropertyChange.getValue());
                            // changed on both sides, ask conflict processor to resolve the issue or raise conflict error
                            RevisionPropertyDiff resolution = conflictProcessor.handleChangedInSourceAndTarget(changedInSourceAndTargetId, mapping, sourceChangeDiff, targetChangeDiff, mapper);
                            if (resolution == null) {
                                conflicts.add(new ChangedInSourceAndTargetConflict(sourcePropertyChange.getObject(), sourceChangeDiff.convert(conflictProcessor), targetChangeDiff.convert(conflictProcessor)));
                            } else {
                                if (!propertyUpdatesToApply.containsKey(type)) {
                                    propertyUpdatesToApply.put(type, HashMultimap.create());
                                }
                                propertyUpdatesToApply.get(type).put(changedInSourceAndTargetId, resolution);
                            }
                            fromChangeSet.removeChanged(type, changedInSourceAndTargetId);
                        }
                    }
                } else {
                    fromChangeSet.removeChanged(type, changedInSourceAndTargetId);
                }
                // this object has changed on both sides either by tracked field changes or due to some cascading derived field change
                // revise the revision on source, since we already have one on this branch already
                revisionsToReviseOnMergeSource.put(type, changedInSourceAndTargetId);
            }
        }
    }
    // after generic conflict processing execute domain specific merge rules via conflict processor
    conflictProcessor.checkConflicts(this, fromChangeSet, toChangeSet).forEach(conflicts::add);
    // handle domain-specific conflict filtering, like donated content, etc.
    // and add all reported conflicts to conflictsToReport
    conflictsToReport.addAll(conflictProcessor.filterConflicts(this, conflicts));
}

Example 5 with DocumentMapping

use of com.b2international.index.mapping.DocumentMapping in project snow-owl by b2ihealthcare.

the class EsIndexAdmin method updateSettings.

@Override
public void updateSettings(Map<String, Object> newSettings) {
    if (CompareUtils.isEmpty(newSettings)) {
        return;
    }
    // ignore local and/or dynamic settings
    final Set<String> unsupportedDynamicSettings = Sets.difference(Sets.difference(newSettings.keySet(), DYNAMIC_SETTINGS), LOCAL_SETTINGS);
    if (!unsupportedDynamicSettings.isEmpty()) {
        throw new IndexException(String.format("Settings [%s] are not dynamically updateable settings.", unsupportedDynamicSettings), null);
    }
    boolean shouldUpdate = false;
    for (String settingKey : newSettings.keySet()) {
        Object currentValue = settings.get(settingKey);
        Object newValue = newSettings.get(settingKey);
        if (!Objects.equals(currentValue, newValue)) {
            shouldUpdate = true;
        }
    }
    if (!shouldUpdate) {
        return;
    }
    Map<String, Object> esSettings = new HashMap<>(newSettings);
    // remove any local settings from esSettings
    esSettings.keySet().removeAll(LOCAL_SETTINGS);
    for (DocumentMapping mapping : mappings.getMappings()) {
        final String index = getTypeIndex(mapping);
        // if any index exists, then update the settings based on the new settings
        if (exists(mapping)) {
            try {
                log.info("Applying settings '{}' changes in index {}...", esSettings, index);
                AcknowledgedResponse response = client.indices().updateSettings(new UpdateSettingsRequest().indices(index).settings(esSettings));
                checkState(response.isAcknowledged(), "Failed to update index settings '%s'.", index);
            } catch (IOException e) {
                throw new IndexException(String.format("Couldn't update settings of index '%s'", index), e);
            }
        }
    }
    // update both local and es settings
    settings.putAll(newSettings);
}