use of org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats in project elasticsearch by elastic.
the class ExtendedStatsIT method testEmptySubAggregation.
public void testEmptySubAggregation() {
SearchResponse searchResponse = client().prepareSearch("idx").setQuery(matchAllQuery()).addAggregation(terms("value").field("value").subAggregation(missing("values").field("values").subAggregation(extendedStats("stats").field("value")))).execute().actionGet();
assertHitCount(searchResponse, 10);
Terms terms = searchResponse.getAggregations().get("value");
assertThat(terms, notNullValue());
assertThat(terms.getBuckets().size(), equalTo(10));
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket.getDocCount(), equalTo(1L));
Missing missing = bucket.getAggregations().get("values");
assertThat(missing, notNullValue());
assertThat(missing.getDocCount(), equalTo(0L));
ExtendedStats stats = missing.getAggregations().get("stats");
assertThat(stats, notNullValue());
assertThat(stats.getName(), equalTo("stats"));
assertThat(stats.getSumOfSquares(), equalTo(0.0));
assertThat(stats.getCount(), equalTo(0L));
assertThat(stats.getSum(), equalTo(0.0));
assertThat(stats.getMin(), equalTo(Double.POSITIVE_INFINITY));
assertThat(stats.getMax(), equalTo(Double.NEGATIVE_INFINITY));
assertThat(Double.isNaN(stats.getStdDeviation()), is(true));
assertThat(Double.isNaN(stats.getAvg()), is(true));
assertThat(Double.isNaN(stats.getStdDeviationBound(ExtendedStats.Bounds.UPPER)), is(true));
assertThat(Double.isNaN(stats.getStdDeviationBound(ExtendedStats.Bounds.LOWER)), is(true));
}
}
use of org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats in project elasticsearch by elastic.
the class StringTermsIT method testSingleValuedFieldOrderedByStatsAggAscWithTermsSubAgg.
public void testSingleValuedFieldOrderedByStatsAggAscWithTermsSubAgg() throws Exception {
boolean asc = true;
SearchResponse response = client().prepareSearch("idx").setTypes("type").addAggregation(terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.aggregation("stats.sum_of_squares", asc)).subAggregation(extendedStats("stats").field("i")).subAggregation(terms("subTerms").field("s_values").collectMode(randomFrom(SubAggCollectionMode.values())))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
ExtendedStats stats = bucket.getAggregations().get("stats");
assertThat(stats, notNullValue());
assertThat(stats.getMax(), equalTo((double) i));
Terms subTermsAgg = bucket.getAggregations().get("subTerms");
assertThat(subTermsAgg, notNullValue());
assertThat(subTermsAgg.getBuckets().size(), equalTo(2));
int j = i;
for (Terms.Bucket subBucket : subTermsAgg.getBuckets()) {
assertThat(subBucket, notNullValue());
assertThat(key(subBucket), equalTo("val" + j));
assertThat(subBucket.getDocCount(), equalTo(1L));
j++;
}
i++;
}
}
use of org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats in project elasticsearch by elastic.
the class StringTermsIT method testSingleValuedFieldOrderedByMultiValueExtendedStatsAsc.
public void testSingleValuedFieldOrderedByMultiValueExtendedStatsAsc() throws Exception {
boolean asc = true;
SearchResponse response = client().prepareSearch("idx").setTypes("type").addAggregation(terms("terms").executionHint(randomExecutionHint()).field(SINGLE_VALUED_FIELD_NAME).collectMode(randomFrom(SubAggCollectionMode.values())).order(Terms.Order.aggregation("stats.sum_of_squares", asc)).subAggregation(extendedStats("stats").field("i"))).execute().actionGet();
assertSearchResponse(response);
Terms terms = response.getAggregations().get("terms");
assertThat(terms, notNullValue());
assertThat(terms.getName(), equalTo("terms"));
assertThat(terms.getBuckets().size(), equalTo(5));
int i = 0;
for (Terms.Bucket bucket : terms.getBuckets()) {
assertThat(bucket, notNullValue());
assertThat(key(bucket), equalTo("val" + i));
assertThat(bucket.getDocCount(), equalTo(1L));
ExtendedStats stats = bucket.getAggregations().get("stats");
assertThat(stats, notNullValue());
assertThat(stats.getMax(), equalTo((double) i));
i++;
}
}
use of org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats in project stash-codesearch-plugin by palantir.
the class SearchServlet method doGet.
@Override
protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException {
// Make sure user is logged in
try {
validationService.validateAuthenticated();
} catch (AuthorisationException notLoggedInException) {
try {
resp.sendRedirect(propertiesService.getLoginUri(URI.create(req.getRequestURL() + (req.getQueryString() == null ? "" : "?" + req.getQueryString()))).toASCIIString());
} catch (Exception e) {
log.error("Unable to redirect unauthenticated user to login page", e);
}
return;
}
// Query and parse settings
SearchParams params = SearchParams.getParams(req, DateTimeZone.forTimeZone(propertiesService.getDefaultTimeZone()));
GlobalSettings globalSettings = settingsManager.getGlobalSettings();
ImmutableSet.Builder<String> noHighlightBuilder = new ImmutableSet.Builder<String>();
for (String extension : globalSettings.getNoHighlightExtensions().split(",")) {
extension = extension.trim().toLowerCase();
if (!extension.isEmpty()) {
noHighlightBuilder.add(extension);
}
}
ImmutableSet<String> noHighlight = noHighlightBuilder.build();
int maxPreviewLines = globalSettings.getMaxPreviewLines();
int maxMatchLines = globalSettings.getMaxMatchLines();
int maxFragments = globalSettings.getMaxFragments();
int pageSize = globalSettings.getPageSize();
TimeValue searchTimeout = new TimeValue(globalSettings.getSearchTimeout());
float commitHashBoost = (float) globalSettings.getCommitHashBoost();
float commitSubjectBoost = (float) globalSettings.getCommitBodyBoost();
float commitBodyBoost = (float) globalSettings.getCommitBodyBoost();
float fileNameBoost = (float) globalSettings.getFileNameBoost();
// Execute ES query
int pages = 0;
long totalHits = 0;
long searchTime = 0;
SearchHit[] currentHits = {};
String error = "";
ArrayList<ImmutableMap<String, Object>> hitArray = new ArrayList<ImmutableMap<String, Object>>(currentHits.length);
ImmutableMap<String, Object> statistics = ImmutableMap.of();
if (params.doSearch) {
// Repo map is null iff user is a system administrator (don't need to validate permissions).
ImmutableMap<String, Repository> repoMap;
try {
validationService.validateForGlobal(Permission.SYS_ADMIN);
repoMap = null;
} catch (AuthorisationException e) {
repoMap = repositoryServiceManager.getRepositoryMap(validationService);
if (repoMap.isEmpty()) {
error = "You do not have permissions to access any repositories";
}
}
int startIndex = params.page * pageSize;
SearchRequestBuilder esReq = es.getClient().prepareSearch(ES_SEARCHALIAS).setFrom(startIndex).setSize(pageSize).setTimeout(searchTimeout).setFetchSource(true);
if (error != null && !error.isEmpty()) {
log.warn("Not performing search due to error {}", error);
} else {
// Build query source and perform query
QueryBuilder query = matchAllQuery();
if (params.searchString != null && !params.searchString.isEmpty()) {
QueryStringQueryBuilder queryStringQuery = queryString(params.searchString).analyzeWildcard(true).lenient(true).defaultOperator(QueryStringQueryBuilder.Operator.AND);
if (params.searchCommits) {
queryStringQuery.field("commit.subject", commitSubjectBoost).field("commit.hash", commitHashBoost).field("commit.body", commitBodyBoost);
}
if (params.searchFilenames) {
queryStringQuery.field("file.path", fileNameBoost);
}
if (params.searchCode) {
queryStringQuery.field("file.contents", 1);
}
query = queryStringQuery;
}
FilterBuilder filter = andFilter(boolFilter().must(repoMap == null ? matchAllFilter() : sf.aclFilter(repoMap), sf.refFilter(params.refNames.split(",")), sf.projectFilter(params.projectKeys.split(",")), sf.repositoryFilter(params.repoNames.split(",")), sf.extensionFilter(params.extensions.split(",")), sf.authorFilter(params.authorNames.split(","))), sf.dateRangeFilter(params.committedAfter, params.committedBefore));
FilteredQueryBuilder finalQuery = filteredQuery(query, filter);
esReq.setQuery(finalQuery).setHighlighterPreTags("").setHighlighterPostTags("").addHighlightedField("contents", 1, maxFragments);
String[] typeArray = {};
if (params.searchCommits) {
if (params.searchFilenames || params.searchCode) {
typeArray = new String[] { "commit", "file" };
} else {
typeArray = new String[] { "commit" };
}
} else if (params.searchFilenames || params.searchCode) {
typeArray = new String[] { "file" };
}
esReq.setTypes(typeArray);
// Build aggregations if statistics were requested
if (params.showStatistics) {
esReq.addAggregation(cardinality("authorCardinality").field("authoremail.untouched").precisionThreshold(1000)).addAggregation(terms("authorRanking").field("authoremail.untouched").size(25)).addAggregation(percentiles("charcountPercentiles").field("charcount").percentiles(PERCENTILES)).addAggregation(extendedStats("charcountStats").field("charcount")).addAggregation(filter("commitCount").filter(typeFilter("commit"))).addAggregation(cardinality("extensionCardinality").field("extension").precisionThreshold(1000)).addAggregation(terms("extensionRanking").field("extension").size(25)).addAggregation(percentiles("linecountPercentiles").field("linecount").percentiles(PERCENTILES)).addAggregation(extendedStats("linecountStats").field("linecount"));
}
SearchResponse esResp = null;
try {
esResp = esReq.get();
} catch (SearchPhaseExecutionException e) {
log.warn("Query failure", e);
error = "Make sure your query conforms to the Lucene/Elasticsearch query string syntax.";
}
if (esResp != null) {
SearchHits esHits = esResp.getHits();
totalHits = esHits.getTotalHits();
pages = (int) Math.min(Integer.MAX_VALUE, (totalHits + pageSize - 1) / pageSize);
currentHits = esHits.getHits();
searchTime = esResp.getTookInMillis();
for (ShardSearchFailure failure : esResp.getShardFailures()) {
log.warn("Shard failure {}", failure.reason());
if (error == null || error.isEmpty()) {
error = "Shard failure: " + failure.reason();
}
}
Aggregations aggs = esResp.getAggregations();
if (params.showStatistics && aggs != null && !aggs.asList().isEmpty()) {
Cardinality authorCardinality = aggs.get("authorCardinality");
Terms authorRanking = aggs.get("authorRanking");
Percentiles charcountPercentiles = aggs.get("charcountPercentiles");
Filter commitCount = aggs.get("commitCount");
ExtendedStats charcountStats = aggs.get("charcountStats");
Cardinality extensionCardinality = aggs.get("extensionCardinality");
Terms extensionRanking = aggs.get("extensionRanking");
Percentiles linecountPercentiles = aggs.get("linecountPercentiles");
ExtendedStats linecountStats = aggs.get("linecountStats");
statistics = new ImmutableMap.Builder<String, Object>().put("authorCardinality", authorCardinality.getValue()).put("authorRanking", getSoyRankingList(authorRanking, commitCount.getDocCount())).put("charcount", new ImmutableMap.Builder<String, Object>().put("average", charcountStats.getAvg()).put("max", Math.round(charcountStats.getMax())).put("min", Math.round(charcountStats.getMin())).put("percentiles", getSoyPercentileList(charcountPercentiles, PERCENTILES)).put("sum", Math.round(charcountStats.getSum())).build()).put("commitcount", commitCount.getDocCount()).put("extensionCardinality", extensionCardinality.getValue()).put("extensionRanking", getSoyRankingList(extensionRanking, charcountStats.getCount())).put("filecount", charcountStats.getCount()).put("linecount", new ImmutableMap.Builder<String, Object>().put("average", linecountStats.getAvg()).put("max", Math.round(linecountStats.getMax())).put("min", Math.round(linecountStats.getMin())).put("percentiles", getSoyPercentileList(linecountPercentiles, PERCENTILES)).put("sum", Math.round(linecountStats.getSum())).build()).build();
}
}
}
// Iterate through current page of search hits
for (SearchHit hit : currentHits) {
ImmutableMap<String, Object> hitData = searchHitToDataMap(hit, repoMap, maxPreviewLines, maxMatchLines, noHighlight);
if (hitData != null) {
hitArray.add(hitData);
}
}
}
// Render page
pbs.assembler().resources().requireContext("com.atlassian.auiplugin:aui-date-picker");
pbs.assembler().resources().requireContext("com.atlassian.auiplugin:aui-experimental-tooltips");
pbs.assembler().resources().requireWebResource("com.palantir.stash.stash-code-search:scs-resources");
resp.setContentType("text/html");
try {
String queryString = req.getQueryString();
String fullUri = req.getRequestURI() + "?" + (queryString == null ? "" : queryString.replaceAll("&?page=\\d*", ""));
ImmutableMap<String, Object> data = new ImmutableMap.Builder<String, Object>().put("pages", pages).put("currentPage", params.page).put("prevParams", params.soyParams).put("doSearch", params.doSearch).put("totalHits", totalHits).put("hitArray", hitArray).put("statistics", statistics).put("error", error).put("fullUri", fullUri).put("baseUrl", propertiesService.getBaseUrl().toASCIIString()).put("resultFrom", Math.min(totalHits, params.page * pageSize + 1)).put("resultTo", Math.min(totalHits, (params.page + 1) * pageSize)).put("searchTime", searchTime).build();
soyTemplateRenderer.render(resp.getWriter(), "com.palantir.stash.stash-code-search:codesearch-soy", "plugin.page.codesearch.searchPage", data);
} catch (Exception e) {
log.error("Error rendering Soy template", e);
}
}
Aggregations