Examples with IndicesOptions org.graylog.shaded.elasticsearch7.org.elasticsearch.action.support.IndicesOptions used on opensource projects

Example 41 with IndicesOptions

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.support.IndicesOptions in project crate by crate.

the class CreateSnapshotPlan method createRequest.

@VisibleForTesting
public static CreateSnapshotRequest createRequest(AnalyzedCreateSnapshot createSnapshot, CoordinatorTxnCtx txnCtx, NodeContext nodeCtx, Row parameters, SubQueryResults subQueryResults, Schemas schemas) {
    Function<? super Symbol, Object> eval = x -> SymbolEvaluator.evaluate(txnCtx, nodeCtx, x, parameters, subQueryResults);
    Settings settings = GenericPropertiesConverter.genericPropertiesToSettings(createSnapshot.properties().map(eval), SnapshotSettings.SETTINGS);
    boolean ignoreUnavailable = IGNORE_UNAVAILABLE.get(settings);
    final HashSet<String> snapshotIndices;
    final HashSet<String> templates = new HashSet<>();
    if (createSnapshot.tables().isEmpty()) {
        for (SchemaInfo schemaInfo : schemas) {
            for (TableInfo tableInfo : schemaInfo.getTables()) {
                // only check for user generated tables
                if (tableInfo instanceof DocTableInfo) {
                    Operation.blockedRaiseException(tableInfo, Operation.READ);
                }
            }
        }
        snapshotIndices = new HashSet<>(AnalyzedCreateSnapshot.ALL_INDICES);
    } else {
        snapshotIndices = new HashSet<>(createSnapshot.tables().size());
        for (Table<Symbol> table : createSnapshot.tables()) {
            DocTableInfo docTableInfo;
            try {
                docTableInfo = (DocTableInfo) schemas.resolveTableInfo(table.getName(), Operation.CREATE_SNAPSHOT, txnCtx.sessionContext().sessionUser(), txnCtx.sessionContext().searchPath());
            } catch (Exception e) {
                if (ignoreUnavailable && e instanceof ResourceUnknownException) {
                    LOGGER.info("Ignore unknown relation '{}' for the '{}' snapshot'", table.getName(), createSnapshot.snapshot());
                    continue;
                } else {
                    throw e;
                }
            }
            if (docTableInfo.isPartitioned()) {
                templates.add(PartitionName.templateName(docTableInfo.ident().schema(), docTableInfo.ident().name()));
            }
            if (table.partitionProperties().isEmpty()) {
                snapshotIndices.addAll(Arrays.asList(docTableInfo.concreteIndices()));
            } else {
                var partitionName = toPartitionName(docTableInfo, Lists2.map(table.partitionProperties(), x -> x.map(eval)));
                if (!docTableInfo.partitions().contains(partitionName)) {
                    if (!ignoreUnavailable) {
                        throw new PartitionUnknownException(partitionName);
                    } else {
                        LOGGER.info("ignoring unknown partition of table '{}' with ident '{}'", partitionName.relationName(), partitionName.ident());
                    }
                } else {
                    snapshotIndices.add(partitionName.asIndexName());
                }
            }
        }
    }
    return new CreateSnapshotRequest(createSnapshot.snapshot().getRepository(), createSnapshot.snapshot().getSnapshotId().getName()).includeGlobalState(createSnapshot.tables().isEmpty()).waitForCompletion(WAIT_FOR_COMPLETION.get(settings)).indices(snapshotIndices.toArray(new String[0])).indicesOptions(IndicesOptions.fromOptions(ignoreUnavailable, true, true, false, IndicesOptions.lenientExpandOpen())).templates(templates.stream().toList()).settings(settings);
}

Example 42 with IndicesOptions

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.support.IndicesOptions in project crate by crate.

the class DeleteAllPartitions method executeOrFail.

@Override
public void executeOrFail(DependencyCarrier executor, PlannerContext plannerContext, RowConsumer consumer, Row params, SubQueryResults subQueryResults) {
    if (partitions.isEmpty()) {
        consumer.accept(InMemoryBatchIterator.of(new Row1(0L), SentinelRow.SENTINEL), null);
    } else {
        /*
             * table is partitioned, in case of concurrent "delete from partitions"
             * it could be that some partitions are already deleted,
             * so ignore it if some are missing
             */
        DeleteIndexRequest deleteIndexRequest = new DeleteIndexRequest(partitions().toArray(new String[0])).indicesOptions(IndicesOptions.lenientExpandOpen());
        executor.transportActionProvider().transportDeleteIndexAction().execute(deleteIndexRequest, new OneRowActionListener<>(consumer, r -> Row1.ROW_COUNT_UNKNOWN));
    }
}

Example 43 with IndicesOptions

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.support.IndicesOptions in project graylog2-server by Graylog2.

the class ClusterAdapterES7 method clusterHealth.

private Optional<ClusterHealthResponse> clusterHealth(Collection<String> indices) {
    final String[] indicesAry = indices.toArray(new String[0]);
    if (!indices.isEmpty() && !indicesExist(indicesAry)) {
        return Optional.empty();
    }
    final ClusterHealthRequest request = new ClusterHealthRequest(indicesAry).timeout(TimeValue.timeValueSeconds(Ints.saturatedCast(requestTimeout.toSeconds()))).indicesOptions(IndicesOptions.lenientExpand());
    try {
        return Optional.of(client.execute((c, requestOptions) -> c.cluster().health(request, requestOptions)));
    } catch (ElasticsearchException e) {
        if (LOG.isDebugEnabled()) {
            LOG.error("{} ({})", e.getMessage(), Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("n/a"), e);
        } else {
            LOG.error("{} ({})", e.getMessage(), Optional.ofNullable(e.getCause()).map(Throwable::getMessage).orElse("n/a"));
        }
        return Optional.empty();
    }
}

Example 44 with IndicesOptions

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.support.IndicesOptions in project graylog2-server by Graylog2.

the class IndicesAdapterES7 method indexRangeStatsOfIndex.

@Override
public IndexRangeStats indexRangeStatsOfIndex(String index) {
    final FilterAggregationBuilder builder = AggregationBuilders.filter("agg", QueryBuilders.existsQuery(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.min("ts_min").field(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.max("ts_max").field(Message.FIELD_TIMESTAMP)).subAggregation(AggregationBuilders.terms("streams").size(Integer.MAX_VALUE).field(Message.FIELD_STREAMS));
    final SearchSourceBuilder query = SearchSourceBuilder.searchSource().aggregation(builder).size(0);
    final SearchRequest request = new SearchRequest().source(query).indices(index).searchType(SearchType.DFS_QUERY_THEN_FETCH).indicesOptions(IndicesOptions.lenientExpandOpen());
    final SearchResponse result = client.execute((c, requestOptions) -> c.search(request, requestOptions), "Couldn't build index range of index " + index);
    if (result.getTotalShards() == 0 || result.getAggregations() == null) {
        throw new IndexNotFoundException("Couldn't build index range of index " + index + " because it doesn't exist.");
    }
    final Filter f = result.getAggregations().get("agg");
    if (f == null) {
        throw new IndexNotFoundException("Couldn't build index range of index " + index + " because it doesn't exist.");
    } else if (f.getDocCount() == 0L) {
        LOG.debug("No documents with attribute \"timestamp\" found in index <{}>", index);
        return IndexRangeStats.EMPTY;
    }
    final Min minAgg = f.getAggregations().get("ts_min");
    final long minUnixTime = new Double(minAgg.getValue()).longValue();
    final DateTime min = new DateTime(minUnixTime, DateTimeZone.UTC);
    final Max maxAgg = f.getAggregations().get("ts_max");
    final long maxUnixTime = new Double(maxAgg.getValue()).longValue();
    final DateTime max = new DateTime(maxUnixTime, DateTimeZone.UTC);
    // make sure we return an empty list, so we can differentiate between old indices that don't have this information
    // and newer ones that simply have no streams.
    final Terms streams = f.getAggregations().get("streams");
    final List<String> streamIds = streams.getBuckets().stream().map(MultiBucketsAggregation.Bucket::getKeyAsString).collect(toList());
    return IndexRangeStats.create(min, max, streamIds);
}

Example 45 with IndicesOptions

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.action.support.IndicesOptions in project graylog2-server by Graylog2.

the class MoreSearchAdapterES7 method eventSearch.

@Override
public MoreSearch.Result eventSearch(String queryString, TimeRange timerange, Set<String> affectedIndices, Sorting sorting, int page, int perPage, Set<String> eventStreams, String filterString, Set<String> forbiddenSourceStreams) {
    final QueryBuilder query = (queryString.isEmpty() || queryString.equals("*")) ? matchAllQuery() : queryStringQuery(queryString).allowLeadingWildcard(allowLeadingWildcard);
    final BoolQueryBuilder filter = boolQuery().filter(query).filter(termsQuery(EventDto.FIELD_STREAMS, eventStreams)).filter(requireNonNull(TimeRangeQueryFactory.create(timerange)));
    if (!isNullOrEmpty(filterString)) {
        filter.filter(queryStringQuery(filterString));
    }
    if (!forbiddenSourceStreams.isEmpty()) {
        // If an event has any stream in "source_streams" that the calling search user is not allowed to access,
        // the event must not be in the search result.
        filter.filter(boolQuery().mustNot(termsQuery(EventDto.FIELD_SOURCE_STREAMS, forbiddenSourceStreams)));
    }
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filter).from((page - 1) * perPage).size(perPage).sort(sorting.getField(), sortOrderMapper.fromSorting(sorting)).trackTotalHits(true);
    final Set<String> indices = affectedIndices.isEmpty() ? Collections.singleton("") : affectedIndices;
    final SearchRequest searchRequest = new SearchRequest(indices.toArray(new String[0])).source(searchSourceBuilder).indicesOptions(INDICES_OPTIONS);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Query:\n{}", searchSourceBuilder.toString(new ToXContent.MapParams(Collections.singletonMap("pretty", "true"))));
        LOG.debug("Execute search: {}", searchRequest.toString());
    }
    final SearchResponse searchResult = client.search(searchRequest, "Unable to perform search query");
    final List<ResultMessage> hits = Streams.stream(searchResult.getHits()).map(ResultMessageFactory::fromSearchHit).collect(Collectors.toList());
    final long total = searchResult.getHits().getTotalHits().value;
    return MoreSearch.Result.builder().results(hits).resultsCount(total).duration(searchResult.getTook().getMillis()).usedIndexNames(affectedIndices).executedQuery(searchSourceBuilder.toString()).build();
}