Search in sources :

Example 21 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchRestDAOV5 method searchObjectIdsViaExpression.

private SearchResult<String> searchObjectIdsViaExpression(String structuredQuery, int start, int size, List<String> sortOptions, String freeTextQuery, String docType) {
    try {
        // Build query
        QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
        if (StringUtils.isNotEmpty(structuredQuery)) {
            Expression expression = Expression.fromString(structuredQuery);
            queryBuilder = expression.getFilterBuilder();
        }
        BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder);
        QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery(freeTextQuery);
        BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery);
        return searchObjectIds(indexName, fq, start, size, sortOptions, docType);
    } catch (Exception e) {
        throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, e.getMessage(), e);
    }
}
Also used : ApplicationException(com.netflix.conductor.core.execution.ApplicationException) Expression(com.netflix.conductor.dao.es5.index.query.parser.Expression) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) ResponseException(org.elasticsearch.client.ResponseException) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException)

Example 22 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchRestDAOV5 method getEventExecutions.

@Override
public List<EventExecution> getEventExecutions(String event) {
    try {
        Expression expression = Expression.fromString("event='" + event + "'");
        QueryBuilder queryBuilder = expression.getFilterBuilder();
        BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder);
        QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery("*");
        BoolQueryBuilder query = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery);
        // Create the searchObjectIdsViaExpression source
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(query);
        searchSourceBuilder.sort(new FieldSortBuilder("created").order(SortOrder.ASC));
        // Generate the actual request to send to ES.
        SearchRequest searchRequest = new SearchRequest(logIndexPrefix + "*");
        searchRequest.types(EVENT_DOC_TYPE);
        searchRequest.source(searchSourceBuilder);
        SearchResponse response = elasticSearchClient.search(searchRequest);
        return mapEventExecutionsResponse(response);
    } catch (Exception e) {
        logger.error("Failed to get executions for event: {}", event, e);
        throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, e.getMessage(), e);
    }
}
Also used : SearchRequest(org.elasticsearch.action.search.SearchRequest) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) Expression(com.netflix.conductor.dao.es5.index.query.parser.Expression) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) ResponseException(org.elasticsearch.client.ResponseException) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) SearchResponse(org.elasticsearch.action.search.SearchResponse)

Example 23 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchRestDAOV5 method updateWorkflow.

@Override
public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {
    if (keys.length != values.length) {
        throw new ApplicationException(ApplicationException.Code.INVALID_INPUT, "Number of keys and values do not match");
    }
    long startTime = Instant.now().toEpochMilli();
    UpdateRequest request = new UpdateRequest(indexName, WORKFLOW_DOC_TYPE, workflowInstanceId);
    Map<String, Object> source = IntStream.range(0, keys.length).boxed().collect(Collectors.toMap(i -> keys[i], i -> values[i]));
    request.doc(source);
    logger.debug("Updating workflow {} with {}", workflowInstanceId, source);
    new RetryUtil<UpdateResponse>().retryOnException(() -> {
        try {
            return elasticSearchClient.update(request);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }, null, null, RETRY_COUNT, "Updating workflow document: " + workflowInstanceId, "updateWorkflow");
    long endTime = Instant.now().toEpochMilli();
    logger.debug("Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId);
    Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime);
    Monitors.recordWorkerQueueSize("indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size());
}
Also used : ResponseException(org.elasticsearch.client.ResponseException) GetResponse(org.elasticsearch.action.get.GetResponse) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) HttpStatus(org.apache.http.HttpStatus) Task(com.netflix.conductor.common.metadata.tasks.Task) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) StringUtils(org.apache.commons.lang3.StringUtils) NByteArrayEntity(org.apache.http.nio.entity.NByteArrayEntity) EntityUtils(org.apache.http.util.EntityUtils) WorkflowSummary(com.netflix.conductor.common.run.WorkflowSummary) PreDestroy(javax.annotation.PreDestroy) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) TypeFactory(com.fasterxml.jackson.databind.type.TypeFactory) UpdateResponse(org.elasticsearch.action.update.UpdateResponse) Workflow(com.netflix.conductor.common.run.Workflow) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) JsonNode(com.fasterxml.jackson.databind.JsonNode) EventExecution(com.netflix.conductor.common.metadata.events.EventExecution) ZoneOffset(java.time.ZoneOffset) NStringEntity(org.apache.http.nio.entity.NStringEntity) DeleteResponse(org.elasticsearch.action.delete.DeleteResponse) MapType(com.fasterxml.jackson.databind.type.MapType) SearchHit(org.elasticsearch.search.SearchHit) Message(com.netflix.conductor.core.events.queue.Message) SearchResult(com.netflix.conductor.common.run.SearchResult) GetRequest(org.elasticsearch.action.get.GetRequest) Trace(com.netflix.conductor.annotations.Trace) TaskExecLog(com.netflix.conductor.common.metadata.tasks.TaskExecLog) TimeZone(java.util.TimeZone) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HttpEntity(org.apache.http.HttpEntity) ContentType(org.apache.http.entity.ContentType) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) Instant(java.time.Instant) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) Response(org.elasticsearch.client.Response) LocalDate(java.time.LocalDate) SortOrder(org.elasticsearch.search.sort.SortOrder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) IntStream(java.util.stream.IntStream) RestClient(org.elasticsearch.client.RestClient) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) XContentType(org.elasticsearch.common.xcontent.XContentType) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) SearchRequest(org.elasticsearch.action.search.SearchRequest) Singleton(javax.inject.Singleton) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) Expression(com.netflix.conductor.dao.es5.index.query.parser.Expression) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) IndexResponse(org.elasticsearch.action.index.IndexResponse) LinkedList(java.util.LinkedList) ExecutorService(java.util.concurrent.ExecutorService) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Logger(org.slf4j.Logger) RetryUtil(com.netflix.conductor.common.utils.RetryUtil) TaskSummary(com.netflix.conductor.common.run.TaskSummary) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DateTime(org.joda.time.DateTime) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) Monitors(com.netflix.conductor.metrics.Monitors) TimeUnit(java.util.concurrent.TimeUnit) IndexDAO(com.netflix.conductor.dao.IndexDAO) ElasticSearchConfiguration(com.netflix.conductor.elasticsearch.ElasticSearchConfiguration) Collections(java.util.Collections) BulkRequest(org.elasticsearch.action.bulk.BulkRequest) InputStream(java.io.InputStream) UpdateResponse(org.elasticsearch.action.update.UpdateResponse) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) IOException(java.io.IOException) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor)

Example 24 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchRestDAOV6 method updateWorkflow.

@Override
public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {
    if (keys.length != values.length) {
        throw new ApplicationException(ApplicationException.Code.INVALID_INPUT, "Number of keys and values do not match");
    }
    long startTime = Instant.now().toEpochMilli();
    String docType = StringUtils.isBlank(docTypeOverride) ? WORKFLOW_DOC_TYPE : docTypeOverride;
    UpdateRequest request = new UpdateRequest(workflowIndexName, docType, workflowInstanceId);
    Map<String, Object> source = IntStream.range(0, keys.length).boxed().collect(Collectors.toMap(i -> keys[i], i -> values[i]));
    request.doc(source);
    logger.debug("Updating workflow {} with {}", workflowInstanceId, source);
    new RetryUtil<UpdateResponse>().retryOnException(() -> {
        try {
            return elasticSearchClient.update(request);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }, null, null, RETRY_COUNT, "Updating workflow document: " + workflowInstanceId, "updateWorkflow");
    long endTime = Instant.now().toEpochMilli();
    logger.debug("Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId);
    Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime);
    Monitors.recordWorkerQueueSize("indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size());
}
Also used : ResponseException(org.elasticsearch.client.ResponseException) GetResponse(org.elasticsearch.action.get.GetResponse) RestClientBuilder(org.elasticsearch.client.RestClientBuilder) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) HttpStatus(org.apache.http.HttpStatus) Task(com.netflix.conductor.common.metadata.tasks.Task) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) StringUtils(org.apache.commons.lang3.StringUtils) NByteArrayEntity(org.apache.http.nio.entity.NByteArrayEntity) EntityUtils(org.apache.http.util.EntityUtils) WorkflowSummary(com.netflix.conductor.common.run.WorkflowSummary) PreDestroy(javax.annotation.PreDestroy) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) TypeFactory(com.fasterxml.jackson.databind.type.TypeFactory) UpdateResponse(org.elasticsearch.action.update.UpdateResponse) Workflow(com.netflix.conductor.common.run.Workflow) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) JsonNode(com.fasterxml.jackson.databind.JsonNode) EventExecution(com.netflix.conductor.common.metadata.events.EventExecution) NStringEntity(org.apache.http.nio.entity.NStringEntity) DeleteResponse(org.elasticsearch.action.delete.DeleteResponse) MapType(com.fasterxml.jackson.databind.type.MapType) SearchHit(org.elasticsearch.search.SearchHit) Message(com.netflix.conductor.core.events.queue.Message) SearchResult(com.netflix.conductor.common.run.SearchResult) GetRequest(org.elasticsearch.action.get.GetRequest) Trace(com.netflix.conductor.annotations.Trace) TaskExecLog(com.netflix.conductor.common.metadata.tasks.TaskExecLog) TimeZone(java.util.TimeZone) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HttpEntity(org.apache.http.HttpEntity) ContentType(org.apache.http.entity.ContentType) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) Instant(java.time.Instant) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) Response(org.elasticsearch.client.Response) LocalDate(java.time.LocalDate) SortOrder(org.elasticsearch.search.sort.SortOrder) ParserException(com.netflix.conductor.elasticsearch.query.parser.ParserException) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) IntStream(java.util.stream.IntStream) RestClient(org.elasticsearch.client.RestClient) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) XContentType(org.elasticsearch.common.xcontent.XContentType) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) SearchRequest(org.elasticsearch.action.search.SearchRequest) Singleton(javax.inject.Singleton) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) IndexResponse(org.elasticsearch.action.index.IndexResponse) LinkedList(java.util.LinkedList) ExecutorService(java.util.concurrent.ExecutorService) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Logger(org.slf4j.Logger) RetryUtil(com.netflix.conductor.common.utils.RetryUtil) TaskSummary(com.netflix.conductor.common.run.TaskSummary) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DateTime(org.joda.time.DateTime) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) Monitors(com.netflix.conductor.metrics.Monitors) TimeUnit(java.util.concurrent.TimeUnit) IndexDAO(com.netflix.conductor.dao.IndexDAO) ElasticSearchConfiguration(com.netflix.conductor.elasticsearch.ElasticSearchConfiguration) Collections(java.util.Collections) BulkRequest(org.elasticsearch.action.bulk.BulkRequest) InputStream(java.io.InputStream) UpdateResponse(org.elasticsearch.action.update.UpdateResponse) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) IOException(java.io.IOException) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor)

Example 25 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchDAOV6 method search.

private SearchResult<String> search(String structuredQuery, int start, int size, List<String> sortOptions, String freeTextQuery, String docType) {
    try {
        docType = StringUtils.isBlank(docTypeOverride) ? docType : docTypeOverride;
        BoolQueryBuilder fq = boolQueryBuilder(structuredQuery, freeTextQuery);
        final SearchRequestBuilder srb = elasticSearchClient.prepareSearch(getIndexName(docType)).setQuery(fq).setTypes(docType).storedFields("_id").setFrom(start).setSize(size);
        addSortOptions(srb, sortOptions);
        return mapSearchResult(srb.get());
    } catch (ParserException e) {
        throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, e.getMessage(), e);
    }
}
Also used : ParserException(com.netflix.conductor.elasticsearch.query.parser.ParserException) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) SearchRequestBuilder(org.elasticsearch.action.search.SearchRequestBuilder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder)

Aggregations

ApplicationException (com.netflix.conductor.core.execution.ApplicationException)93 Task (com.netflix.conductor.common.metadata.tasks.Task)22 Workflow (com.netflix.conductor.common.run.Workflow)22 HashMap (java.util.HashMap)19 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)16 IOException (java.io.IOException)16 Map (java.util.Map)15 ArrayList (java.util.ArrayList)14 List (java.util.List)14 Inject (javax.inject.Inject)14 Singleton (javax.inject.Singleton)14 Logger (org.slf4j.Logger)14 LoggerFactory (org.slf4j.LoggerFactory)14 Trace (com.netflix.conductor.annotations.Trace)13 EventExecution (com.netflix.conductor.common.metadata.events.EventExecution)13 Monitors (com.netflix.conductor.metrics.Monitors)13 Collectors (java.util.stream.Collectors)13 ResultSet (com.datastax.driver.core.ResultSet)12 EventHandler (com.netflix.conductor.common.metadata.events.EventHandler)12 TimeUnit (java.util.concurrent.TimeUnit)12