use of com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient in project vind by RBMHTechnology.
the class ElasticQueryBuilder method buildElasticAggregations.
private static List<AggregationBuilder> buildElasticAggregations(String name, Facet vindFacet, DocumentFactory factory, String searchContext, int minCount, int facetLimit, List<String> indexFootPrint, ElasticVindClient client, SearchSourceBuilder searchSourceStatic) {
final String contextualizedFacetName = Stream.of(searchContext, name).filter(Objects::nonNull).collect(Collectors.joining("_"));
final UseCase facetScope = UseCase.valueOf(vindFacet.getScope().name());
switch(vindFacet.getType()) {
case "TermFacet":
final Facet.TermFacet termFacet = (Facet.TermFacet) vindFacet;
final FieldDescriptor<?> field = factory.getField(termFacet.getFieldName());
if (field.hasUseCase(facetScope)) {
final String fieldName = FieldUtil.getFieldName(field, facetScope, searchContext, indexFootPrint).orElse(termFacet.getFieldName());
final TermsAggregationBuilder termsAgg = AggregationBuilders.terms(contextualizedFacetName).field(fieldName).minDocCount(minCount).size(facetLimit);
Optional.ofNullable(termFacet.getOption()).ifPresent(option -> setTermOptions(termsAgg, option));
addSortToAggregation(vindFacet, searchContext, termsAgg, indexFootPrint);
return Collections.singletonList(termsAgg);
} else
return Collections.emptyList();
case "TypeFacet":
final Facet.TypeFacet typeFacet = (Facet.TypeFacet) vindFacet;
final TermsAggregationBuilder typeAgg = AggregationBuilders.terms(name).field(FieldUtil.TYPE).minDocCount(minCount).size(facetLimit);
addSortToAggregation(vindFacet, searchContext, typeAgg, indexFootPrint);
return Collections.singletonList(typeAgg);
case "QueryFacet":
final List<AggregationBuilder> aggs = new ArrayList<>();
final Facet.QueryFacet queryFacet = (Facet.QueryFacet) vindFacet;
final Filter vindFilter = queryFacet.getFilter();
Optional.ofNullable(filterMapper(vindFilter, factory, searchContext, indexFootPrint)).ifPresent(elasticFilter -> aggs.add(AggregationBuilders.filters(contextualizedFacetName, elasticFilter)));
return aggs;
case "NumericRangeFacet":
final Facet.NumericRangeFacet<?> numericRangeFacet = (Facet.NumericRangeFacet) vindFacet;
final Optional<String> numericRangeAggField = FieldUtil.getFieldName(numericRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (numericRangeAggField.isPresent()) {
final HistogramAggregationBuilder rangeAggregation = AggregationBuilders.histogram(name).keyed(true).field(numericRangeAggField.get()).interval(numericRangeFacet.getGap().doubleValue()).minDocCount(minCount);
addSortToAggregation(vindFacet, searchContext, rangeAggregation, indexFootPrint);
final RangeAggregationBuilder numericIntervalRangeAggregation = AggregationBuilders.range(contextualizedFacetName).keyed(true).field(numericRangeAggField.get()).subAggregation(rangeAggregation);
numericIntervalRangeAggregation.addRange(numericRangeFacet.getStart().doubleValue(), numericRangeFacet.getEnd().doubleValue());
return Collections.singletonList(numericIntervalRangeAggregation);
}
return Collections.emptyList();
case "ZoneDateRangeFacet":
final Facet.DateRangeFacet.ZoneDateRangeFacet zoneDateRangeFacet = (Facet.DateRangeFacet.ZoneDateRangeFacet) vindFacet;
final Optional<String> zoneDateRangeAggFieldName = FieldUtil.getFieldName(zoneDateRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (zoneDateRangeAggFieldName.isPresent()) {
final DateRangeAggregationBuilder dateRangeAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(zoneDateRangeAggFieldName.get());
final ZonedDateTime zonedDateTimeEnd = (ZonedDateTime) zoneDateRangeFacet.getEnd();
final ZonedDateTime zonedDateTimeStart = (ZonedDateTime) zoneDateRangeFacet.getStart();
dateRangeAggregation.addRange(zonedDateTimeStart, zonedDateTimeEnd);
final DateHistogramAggregationBuilder histogramDateRangeAggregation = AggregationBuilders.dateHistogram(name).minDocCount(minCount).fixedInterval(DateHistogramInterval.minutes(new Long(zoneDateRangeFacet.getGapDuration().toMinutes()).intValue())).keyed(true).field(zoneDateRangeAggFieldName.get());
addSortToAggregation(vindFacet, searchContext, histogramDateRangeAggregation, indexFootPrint);
dateRangeAggregation.subAggregation(histogramDateRangeAggregation);
return Collections.singletonList(dateRangeAggregation);
}
return Collections.emptyList();
case "UtilDateRangeFacet":
final Facet.DateRangeFacet.UtilDateRangeFacet utilDateRangeFacet = (Facet.DateRangeFacet.UtilDateRangeFacet) vindFacet;
final Optional<String> utilDateRangeAggFieldName = FieldUtil.getFieldName(utilDateRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (utilDateRangeAggFieldName.isPresent()) {
final DateRangeAggregationBuilder utilDateRangeAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(utilDateRangeAggFieldName.get());
final ZonedDateTime dateTimeEnd = ZonedDateTime.ofInstant(((Date) utilDateRangeFacet.getEnd()).toInstant(), ZoneId.of("UTC"));
final ZonedDateTime dateTimeStart = ZonedDateTime.ofInstant(((Date) utilDateRangeFacet.getStart()).toInstant(), ZoneId.of("UTC"));
utilDateRangeAggregation.addRange(dateTimeStart, dateTimeEnd);
final DateHistogramAggregationBuilder histogramUtilDateRangeAggregation = AggregationBuilders.dateHistogram(name).keyed(true).fixedInterval(DateHistogramInterval.minutes(new Long(utilDateRangeFacet.getGapDuration().toMinutes()).intValue())).field(utilDateRangeAggFieldName.get());
addSortToAggregation(vindFacet, searchContext, histogramUtilDateRangeAggregation, indexFootPrint);
utilDateRangeAggregation.subAggregation(histogramUtilDateRangeAggregation);
return Collections.singletonList(utilDateRangeAggregation);
}
return Collections.emptyList();
case "DateMathRangeFacet":
final Facet.DateRangeFacet.DateMathRangeFacet dateMathRangeFacet = (Facet.DateRangeFacet.DateMathRangeFacet) vindFacet;
final Optional<String> dateMathRangeAggFieldName = FieldUtil.getFieldName(dateMathRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (dateMathRangeAggFieldName.isPresent()) {
final DateRangeAggregationBuilder dateMathDateRangeAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(dateMathRangeAggFieldName.get());
final ZonedDateTime dateMathEnd = ZonedDateTime.ofInstant(Instant.ofEpochSecond(((DateMathExpression) dateMathRangeFacet.getEnd()).getTimeStamp()), ZoneId.of("UTC"));
final ZonedDateTime dateMathStart = ZonedDateTime.ofInstant(Instant.ofEpochSecond(((DateMathExpression) dateMathRangeFacet.getStart()).getTimeStamp()), ZoneId.of("UTC"));
dateMathDateRangeAggregation.addRange(name, ((DateMathExpression) dateMathRangeFacet.getStart()).toElasticString(), ((DateMathExpression) dateMathRangeFacet.getEnd()).toElasticString());
final Long minutesGap = dateMathRangeFacet.getGapDuration().toMinutes();
final DateHistogramAggregationBuilder histogramDateMathDateRangeAggregation = AggregationBuilders.dateHistogram(name).minDocCount(minCount).fixedInterval(DateHistogramInterval.minutes(minutesGap.intValue())).keyed(true).field(dateMathRangeAggFieldName.get());
dateMathDateRangeAggregation.subAggregation(histogramDateMathDateRangeAggregation);
addSortToAggregation(vindFacet, searchContext, histogramDateMathDateRangeAggregation, indexFootPrint);
return Collections.singletonList(dateMathDateRangeAggregation);
}
return Collections.emptyList();
case "NumericIntervalFacet":
final Facet.NumericIntervalFacet numericIntervalFacet = (Facet.NumericIntervalFacet) vindFacet;
final Optional<String> numericIntervalAggFieldName = FieldUtil.getFieldName(numericIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (numericIntervalAggFieldName.isPresent()) {
final RangeAggregationBuilder numericIntervalAggregation = AggregationBuilders.range(contextualizedFacetName).keyed(true).field(numericIntervalAggFieldName.get());
numericIntervalFacet.getIntervals().forEach(interval -> intervalToRange((NumericInterval<?>) interval, numericIntervalAggregation));
return Collections.singletonList(numericIntervalAggregation);
}
return Collections.emptyList();
case "ZoneDateTimeIntervalFacet":
final Facet.DateIntervalFacet.ZoneDateTimeIntervalFacet zoneDateTimeIntervalFacet = (Facet.DateIntervalFacet.ZoneDateTimeIntervalFacet) vindFacet;
final Optional<String> zoneDateTimeIntervalAggFieldName = FieldUtil.getFieldName(zoneDateTimeIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (zoneDateTimeIntervalAggFieldName.isPresent()) {
final DateRangeAggregationBuilder ZoneDateIntervalAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(zoneDateTimeIntervalAggFieldName.get());
zoneDateTimeIntervalFacet.getIntervals().forEach(interval -> intervalToRange((Interval.ZonedDateTimeInterval<?>) interval, ZoneDateIntervalAggregation));
return Collections.singletonList(ZoneDateIntervalAggregation);
}
return Collections.emptyList();
case "UtilDateIntervalFacet":
final Facet.DateIntervalFacet.UtilDateIntervalFacet utilDateIntervalFacet = (Facet.DateIntervalFacet.UtilDateIntervalFacet) vindFacet;
final Optional<String> utilDateIntervalAggFieldName = FieldUtil.getFieldName(utilDateIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (utilDateIntervalAggFieldName.isPresent()) {
final DateRangeAggregationBuilder utilDateIntervalAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(utilDateIntervalAggFieldName.get());
utilDateIntervalFacet.getIntervals().forEach(interval -> intervalToRange((Interval.UtilDateInterval<?>) interval, utilDateIntervalAggregation));
return Collections.singletonList(utilDateIntervalAggregation);
}
return Collections.emptyList();
case "ZoneDateTimeDateMathIntervalFacet":
case "UtilDateMathIntervalFacet":
final Facet.DateIntervalFacet dateMathIntervalFacet = (Facet.DateIntervalFacet) vindFacet;
final Optional<String> dateMathIntervalAggFieldName = FieldUtil.getFieldName(dateMathIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
if (dateMathIntervalAggFieldName.isPresent()) {
final DateRangeAggregationBuilder dateMathIntervalAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(dateMathIntervalAggFieldName.get());
dateMathIntervalFacet.getIntervals().forEach(interval -> intervalToRange((Interval.DateMathInterval<?>) interval, dateMathIntervalAggregation));
return Collections.singletonList(dateMathIntervalAggregation);
}
return Collections.emptyList();
case "StatsDateFacet":
case "StatsUtilDateFacet":
case "StatsNumericFacet":
case "StatsFacet":
final Facet.StatsFacet statsFacet = (Facet.StatsFacet) vindFacet;
if (String.class.isAssignableFrom(FieldUtil.getFieldType(statsFacet.getField(), facetScope))) {
return getStringStatsAggregationBuilders(searchContext, contextualizedFacetName, facetScope, statsFacet, indexFootPrint);
}
return getStatsAggregationBuilders(searchContext, contextualizedFacetName, facetScope, statsFacet, indexFootPrint);
case "PivotFacet":
final Facet.PivotFacet pivotFacet = (Facet.PivotFacet) vindFacet;
final int facetSize = pivotFacet.getSize().orElse(facetLimit);
final List<TermsAggregationBuilder> termFacets = pivotFacet.getBuckets().stream().map(bucket -> Pair.of(FieldUtil.getFieldName(bucket.getLeft(), facetScope, searchContext, indexFootPrint), bucket.getRight())).filter(bucket -> bucket.getLeft().isPresent()).map(bucket -> AggregationBuilders.terms(contextualizedFacetName).field(bucket.getLeft().get()).minDocCount(minCount).size(Optional.ofNullable(bucket.getRight()).orElse(facetSize))).collect(Collectors.toList());
termFacets.forEach(agg -> addSortToAggregation(vindFacet, searchContext, agg, indexFootPrint));
final TermsAggregationBuilder mainBucket = termFacets.get(0);
if (pivotFacet.getPage().isPresent()) {
try {
final int fieldCardinality = getFieldCardinality(client, mainBucket.field(), searchSourceStatic);
final int partitions = (int) Math.ceil(fieldCardinality / Double.valueOf(mainBucket.size()));
mainBucket.includeExclude(new IncludeExclude(Math.toIntExact(pivotFacet.getPage().get()), partitions));
} catch (IOException e) {
throw new SearchServerException("Error calculating cardinality of field" + mainBucket.field() + " for paged pivot facet: " + e.getMessage(), e);
}
}
final Optional<TermsAggregationBuilder> pivotAgg = Lists.reverse(termFacets).stream().reduce((agg1, agg2) -> agg2.subAggregation(agg1));
return Collections.singletonList(pivotAgg.orElse(null));
default:
throw new SearchServerException(String.format("Error mapping Vind facet to Elasticsearch aggregation: Unknown facet type %s", vindFacet.getType()));
}
}
use of com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient in project vind by RBMHTechnology.
the class ElasticQueryBuilder method buildQuery.
public static SearchSourceBuilder buildQuery(FulltextSearch search, DocumentFactory factory, boolean escape, List<String> indexFootPrint, ElasticVindClient client) {
final String searchContext = search.getSearchContext();
final SearchSourceBuilder searchSource = new SearchSourceBuilder();
final BoolQueryBuilder baseQuery = QueryBuilders.boolQuery();
// Set total hits count
final boolean trackTotalHits = Boolean.parseBoolean(SearchConfiguration.get(SearchConfiguration.TRACK_TOTAL_HITS, "true"));
searchSource.trackTotalHits(trackTotalHits);
// build full text disMax query
String searchString = "*".equals(search.getSearchString()) || Strings.isEmpty(search.getSearchString().trim()) ? "*:*" : search.getSearchString().trim();
if (escape) {
// Escape especial characters: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
for (Map.Entry<String, String> wordEntry : reservedChars.entrySet()) {
searchString = searchString.replaceAll(wordEntry.getKey(), wordEntry.getValue());
}
}
if (!escape && searchString.contains(":")) {
String skipColonSearchString = searchString.replaceAll(":", reservedChars.get(":"));
final String[] skipedFields = Arrays.stream(skipColonSearchString.split(" ")).filter(term -> term.contains("\\:")).map(term -> term.substring(0, term.indexOf("\\:"))).filter(posibleField -> indexFootPrint.contains(posibleField) || "*".equals(posibleField)).toArray(String[]::new);
for (String field : skipedFields) {
skipColonSearchString = skipColonSearchString.replace(field + "\\:", field + ":");
}
searchString = skipColonSearchString;
}
final DisMaxQueryBuilder query = createDisMaxQueryBuilder(new FulltextTerm(searchString, search.getMinimumShouldMatch()), factory, indexFootPrint, searchContext);
baseQuery.must(query);
searchSource.query(baseQuery);
if (search.isSpellcheck()) {
final SuggestBuilder suggestBuilder = new SuggestBuilder();
suggestBuilder.setGlobalText(search.getSearchString());
Lists.newArrayList(getFullTextFieldNames(search, factory, searchContext, indexFootPrint)).forEach(fieldName -> suggestBuilder.addSuggestion(FieldUtil.getSourceFieldName(fieldName.replaceAll(".text", ""), searchContext), SuggestBuilders.termSuggestion(fieldName).prefixLength(0)));
searchSource.suggest(suggestBuilder);
}
searchSource.trackScores(SearchConfiguration.get(SearchConfiguration.SEARCH_RESULT_SHOW_SCORE, true));
if (search.getGeoDistance() != null) {
final FieldDescriptor distanceField = factory.getField(search.getGeoDistance().getFieldName());
final Optional<String> distanceFieldName = FieldUtil.getFieldName(distanceField, searchContext, indexFootPrint);
if (Objects.nonNull(distanceField) && distanceFieldName.isPresent()) {
searchSource.scriptField(FieldUtil.DISTANCE, new Script(ScriptType.INLINE, "painless", String.format(Locale.ENGLISH, "if(doc['%s'].size()!=0)" + "doc['%s'].arcDistance(%f,%f);" + "else []", distanceFieldName.get(), distanceFieldName.get(), search.getGeoDistance().getLocation().getLat(), search.getGeoDistance().getLocation().getLng()), Collections.emptyMap()));
}
}
searchSource.fetchSource(true);
baseQuery.filter(buildFilterQuery(search.getFilter(), factory, searchContext, indexFootPrint));
if (search.hasFacet() && search.getFacetLimit() != 0) {
final int facetLimit = search.getFacetLimit() < 0 ? Integer.MAX_VALUE : search.getFacetLimit();
if (search.getFacetLimit() < 0) {
log.info("Facet limit has been set to {}: " + "Elastic search does not support unlimited facet results, setting it to Integer.MAX_VALUE ({})", search.getFacetLimit(), facetLimit);
}
search.getFacets().entrySet().stream().map(vindFacet -> {
return buildElasticAggregations(vindFacet.getKey(), vindFacet.getValue(), factory, searchContext, search.getFacetMinCount(), facetLimit, indexFootPrint, client, searchSource);
}).flatMap(Collection::stream).filter(Objects::nonNull).forEach(searchSource::aggregation);
}
search.getFacets().values().stream().filter(facet -> facet.getType().equals("PivotFacet")).map(pivotFacet -> searchSource.aggregations().getAggregatorFactories().stream().filter(agg -> agg.getName().equals(Stream.of(searchContext, pivotFacet.getFacetName()).filter(Objects::nonNull).collect(Collectors.joining("_")))).collect(Collectors.toList())).flatMap(Collection::stream).forEach(pivotAgg -> {
final List<String> facets = search.getFacets().values().stream().filter(facet -> Arrays.asList(facet.getTagedPivots()).contains(pivotAgg.getName().replaceAll(searchContext + "_", ""))).map(facet -> Stream.of(searchContext, facet.getFacetName()).filter(Objects::nonNull).collect(Collectors.joining("_"))).collect(Collectors.toList());
final List<AggregationBuilder> aggs = searchSource.aggregations().getAggregatorFactories().stream().filter(agg -> facets.contains(agg.getName())).collect(Collectors.toList());
addToPivotAggs(pivotAgg, aggs, indexFootPrint);
});
// sorting
if (search.hasSorting()) {
search.getSorting().stream().map(sort -> SortUtils.buildSort(sort, search, factory, searchContext, indexFootPrint)).forEach(searchSource::sort);
}
// paging
switch(search.getResultSet().getType()) {
case page:
{
final Page resultSet = (Page) search.getResultSet();
searchSource.from(resultSet.getOffset());
searchSource.size(resultSet.getPagesize());
break;
}
case slice:
{
final Slice resultSet = (Slice) search.getResultSet();
searchSource.from(resultSet.getOffset());
searchSource.size(resultSet.getSliceSize());
break;
}
case cursor:
{
final Cursor resultSet = (Cursor) search.getResultSet();
searchSource.size(resultSet.getSize());
searchSource.sort("_id_");
if (resultSet.getSearchAfter() != null) {
searchSource.searchAfter(fromSearchAfterCursor(resultSet.getSearchAfter()));
}
break;
}
}
return searchSource;
}
use of com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient in project vind by RBMHTechnology.
the class ElasticsearchServerProvider method getInstance.
@Override
public ElasticVindClient getInstance() {
final Logger log = LoggerFactory.getLogger(ElasticServerProvider.class);
final String host = SearchConfiguration.get(SearchConfiguration.SERVER_HOST);
if (host == null) {
log.error("{} has to be set", SearchConfiguration.SERVER_HOST);
throw new SearchServerInstantiateException(SearchConfiguration.SERVER_HOST + " has to be set", this.getClass());
}
final String collection = SearchConfiguration.get(SearchConfiguration.SERVER_COLLECTION);
final String connectionTimeout = SearchConfiguration.get(SearchConfiguration.SERVER_CONNECTION_TIMEOUT);
final String soTimeout = SearchConfiguration.get(SearchConfiguration.SERVER_SO_TIMEOUT);
log.info("Instantiating Elasticsearch client: {}", host);
if (collection != null) {
ElasticVindClient client;
final AuthTypes authType = AuthTypes.valueOf(SearchConfiguration.get(SearchConfiguration.SEARCH_AUTHENTICATION_METHOD, AuthTypes.NONE.name()));
final ElasticVindClient.Builder clientBuilder = new ElasticVindClient.Builder(host).setDefaultIndex(collection);
if (StringUtils.isNotEmpty(connectionTimeout)) {
clientBuilder.setConnectionTimeout(Long.parseLong(connectionTimeout));
}
if (StringUtils.isNotEmpty(soTimeout)) {
clientBuilder.setSocketTimeout(Long.parseLong(soTimeout));
}
switch(authType) {
case APIKEY:
final String id = SearchConfiguration.get(SearchConfiguration.SEARCH_API_KEY_ID);
final String key = SearchConfiguration.get(SearchConfiguration.SEARCH_API_KEY_SECRET);
if (Objects.isNull(id) || Objects.isNull(key)) {
log.error("Missing API id or secret to authenticate with Elasticsearch backend");
throw new SearchServerInstantiateException("Missing API id or secret to authenticate with Elasticsearch backend", this.getClass());
}
client = clientBuilder.buildWithApiKeyAuth(id, key);
break;
case BASIC:
final String user = SearchConfiguration.get(SearchConfiguration.SEARCH_AUTHENTICATION_USER);
final String pssw = SearchConfiguration.get(SearchConfiguration.SEARCH_AUTHENTICATION_KEY);
if (Objects.isNull(user) || Objects.isNull(pssw)) {
log.error("Missing API user or password to authenticate with Elasticsearch backend");
throw new SearchServerInstantiateException("Missing API user or password to authenticate with Elasticsearch backend", this.getClass());
}
client = clientBuilder.buildWithBasicAuth(user, pssw);
break;
default:
client = clientBuilder.build();
break;
}
if (StringUtils.isNotEmpty(connectionTimeout)) {
client.setConnectionTimeout(Long.parseLong(connectionTimeout));
}
if (StringUtils.isNotEmpty(soTimeout)) {
client.setClientTimeout(Long.parseLong(soTimeout));
}
return client;
} else {
log.error(SearchConfiguration.SERVER_COLLECTION + " has to be set");
throw new SearchServerInstantiateException(SearchConfiguration.SERVER_COLLECTION + " has to be set", this.getClass());
}
}
Aggregations