Search in sources :

Example 26 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchRestDAOV7 method updateWorkflow.

@Override
public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {
    if (keys.length != values.length) {
        throw new ApplicationException(ApplicationException.Code.INVALID_INPUT, "Number of keys and values do not match");
    }
    long startTime = Instant.now().toEpochMilli();
    UpdateRequest request = new UpdateRequest(workflowIndexName, workflowInstanceId);
    Map<String, Object> source = IntStream.range(0, keys.length).boxed().collect(Collectors.toMap(i -> keys[i], i -> values[i]));
    request.doc(source);
    logger.debug("Updating workflow {} with {}", workflowInstanceId, source);
    new RetryUtil<UpdateResponse>().retryOnException(() -> {
        try {
            return elasticSearchClient.update(request, RequestOptions.DEFAULT);
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
    }, null, null, RETRY_COUNT, "Updating workflow document: " + workflowInstanceId, "updateWorkflow");
    long endTime = Instant.now().toEpochMilli();
    logger.debug("Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId);
    Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime);
    Monitors.recordWorkerQueueSize("indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size());
}
Also used : GetResponse(org.elasticsearch.action.get.GetResponse) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) HttpStatus(org.apache.http.HttpStatus) Task(com.netflix.conductor.common.metadata.tasks.Task) org.elasticsearch.client(org.elasticsearch.client) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) StringUtils(org.apache.commons.lang3.StringUtils) NByteArrayEntity(org.apache.http.nio.entity.NByteArrayEntity) EntityUtils(org.apache.http.util.EntityUtils) WorkflowSummary(com.netflix.conductor.common.run.WorkflowSummary) PreDestroy(javax.annotation.PreDestroy) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) Settings(org.elasticsearch.common.settings.Settings) TypeFactory(com.fasterxml.jackson.databind.type.TypeFactory) UpdateResponse(org.elasticsearch.action.update.UpdateResponse) Workflow(com.netflix.conductor.common.run.Workflow) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) JsonNode(com.fasterxml.jackson.databind.JsonNode) EventExecution(com.netflix.conductor.common.metadata.events.EventExecution) NStringEntity(org.apache.http.nio.entity.NStringEntity) DeleteResponse(org.elasticsearch.action.delete.DeleteResponse) MapType(com.fasterxml.jackson.databind.type.MapType) SearchHit(org.elasticsearch.search.SearchHit) Message(com.netflix.conductor.core.events.queue.Message) SearchResult(com.netflix.conductor.common.run.SearchResult) GetRequest(org.elasticsearch.action.get.GetRequest) Trace(com.netflix.conductor.annotations.Trace) TaskExecLog(com.netflix.conductor.common.metadata.tasks.TaskExecLog) TimeZone(java.util.TimeZone) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HttpEntity(org.apache.http.HttpEntity) ContentType(org.apache.http.entity.ContentType) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) Instant(java.time.Instant) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) LocalDate(java.time.LocalDate) SortOrder(org.elasticsearch.search.sort.SortOrder) ParserException(com.netflix.conductor.elasticsearch.query.parser.ParserException) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) IntStream(java.util.stream.IntStream) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) XContentType(org.elasticsearch.common.xcontent.XContentType) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) SearchRequest(org.elasticsearch.action.search.SearchRequest) Singleton(javax.inject.Singleton) ObjectNode(com.fasterxml.jackson.databind.node.ObjectNode) ArrayList(java.util.ArrayList) Inject(javax.inject.Inject) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) IndexResponse(org.elasticsearch.action.index.IndexResponse) LinkedList(java.util.LinkedList) ExecutorService(java.util.concurrent.ExecutorService) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Logger(org.slf4j.Logger) RetryUtil(com.netflix.conductor.common.utils.RetryUtil) CreateIndexRequest(org.elasticsearch.client.indices.CreateIndexRequest) TaskSummary(com.netflix.conductor.common.run.TaskSummary) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DateTime(org.joda.time.DateTime) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) Monitors(com.netflix.conductor.metrics.Monitors) TimeUnit(java.util.concurrent.TimeUnit) IndexDAO(com.netflix.conductor.dao.IndexDAO) ElasticSearchConfiguration(com.netflix.conductor.elasticsearch.ElasticSearchConfiguration) Collections(java.util.Collections) BulkRequest(org.elasticsearch.action.bulk.BulkRequest) InputStream(java.io.InputStream) UpdateResponse(org.elasticsearch.action.update.UpdateResponse) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) IOException(java.io.IOException) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor)

Example 27 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class QueueManager method startMonitor.

private void startMonitor(Status status, ObservableQueue queue) {
    queue.observe().subscribe((Message msg) -> {
        try {
            logger.debug("Got message {}", msg.getPayload());
            String payload = msg.getPayload();
            JsonNode payloadJSON = objectMapper.readTree(payload);
            String externalId = getValue("externalId", payloadJSON);
            if (externalId == null || "".equals(externalId)) {
                logger.error("No external Id found in the payload {}", payload);
                queue.ack(Collections.singletonList(msg));
                return;
            }
            JsonNode json = objectMapper.readTree(externalId);
            String workflowId = getValue("workflowId", json);
            String taskRefName = getValue("taskRefName", json);
            String taskId = getValue("taskId", json);
            if (workflowId == null || "".equals(workflowId)) {
                // This is a bad message, we cannot process it
                logger.error("No workflow id found in the message. {}", payload);
                queue.ack(Collections.singletonList(msg));
                return;
            }
            Workflow workflow = executionService.getExecutionStatus(workflowId, true);
            Optional<Task> taskOptional;
            if (StringUtils.isNotEmpty(taskId)) {
                taskOptional = workflow.getTasks().stream().filter(task -> !task.getStatus().isTerminal() && task.getTaskId().equals(taskId)).findFirst();
            } else if (StringUtils.isEmpty(taskRefName)) {
                logger.error("No taskRefName found in the message. If there is only one WAIT task, will mark it as completed. {}", payload);
                taskOptional = workflow.getTasks().stream().filter(task -> !task.getStatus().isTerminal() && task.getTaskType().equals(Wait.NAME)).findFirst();
            } else {
                taskOptional = workflow.getTasks().stream().filter(task -> !task.getStatus().isTerminal() && task.getReferenceTaskName().equals(taskRefName)).findFirst();
            }
            if (!taskOptional.isPresent()) {
                logger.error("No matching tasks found to be marked as completed for workflow {}, taskRefName {}, taskId {}", workflowId, taskRefName, taskId);
                queue.ack(Collections.singletonList(msg));
                return;
            }
            Task task = taskOptional.get();
            task.setStatus(status);
            task.getOutputData().putAll(objectMapper.convertValue(payloadJSON, _mapType));
            executionService.updateTask(task);
            List<String> failures = queue.ack(Collections.singletonList(msg));
            if (!failures.isEmpty()) {
                logger.error("Not able to ack the messages {}", failures.toString());
            }
        } catch (JsonParseException e) {
            logger.error("Bad message? : {} ", msg, e);
            queue.ack(Collections.singletonList(msg));
        } catch (ApplicationException e) {
            if (e.getCode().equals(Code.NOT_FOUND)) {
                logger.error("Workflow ID specified is not valid for this environment");
                queue.ack(Collections.singletonList(msg));
            }
            logger.error("Error processing message: {}", msg, e);
        } catch (Exception e) {
            logger.error("Error processing message: {}", msg, e);
        }
    }, (Throwable t) -> {
        logger.error(t.getMessage(), t);
    });
    logger.info("QueueListener::STARTED...listening for " + queue.getName());
}
Also used : Arrays(java.util.Arrays) LoggerFactory(org.slf4j.LoggerFactory) Status(com.netflix.conductor.common.metadata.tasks.Task.Status) HashMap(java.util.HashMap) Wait(com.netflix.conductor.core.execution.tasks.Wait) Singleton(javax.inject.Singleton) Task(com.netflix.conductor.common.metadata.tasks.Task) StringUtils(org.apache.commons.lang3.StringUtils) Inject(javax.inject.Inject) Workflow(com.netflix.conductor.common.run.Workflow) Map(java.util.Map) JsonNode(com.fasterxml.jackson.databind.JsonNode) TypeReference(com.fasterxml.jackson.core.type.TypeReference) JsonParseException(com.fasterxml.jackson.core.JsonParseException) Code(com.netflix.conductor.core.execution.ApplicationException.Code) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) Message(com.netflix.conductor.core.events.queue.Message) Logger(org.slf4j.Logger) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) UUID(java.util.UUID) List(java.util.List) Optional(java.util.Optional) ExecutionService(com.netflix.conductor.service.ExecutionService) Collections(java.util.Collections) ObservableQueue(com.netflix.conductor.core.events.queue.ObservableQueue) Task(com.netflix.conductor.common.metadata.tasks.Task) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) Message(com.netflix.conductor.core.events.queue.Message) Workflow(com.netflix.conductor.common.run.Workflow) JsonNode(com.fasterxml.jackson.databind.JsonNode) JsonParseException(com.fasterxml.jackson.core.JsonParseException) JsonParseException(com.fasterxml.jackson.core.JsonParseException) ApplicationException(com.netflix.conductor.core.execution.ApplicationException)

Example 28 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchDAOV5 method getEventExecutions.

@Override
public List<EventExecution> getEventExecutions(String event) {
    try {
        Expression expression = Expression.fromString("event='" + event + "'");
        QueryBuilder queryBuilder = expression.getFilterBuilder();
        BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder);
        QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery("*");
        BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery);
        final SearchRequestBuilder srb = elasticSearchClient.prepareSearch(logIndexPrefix + "*").setQuery(fq).setTypes(EVENT_DOC_TYPE).addSort(SortBuilders.fieldSort("created").order(SortOrder.ASC));
        return mapEventExecutionsResponse(srb.execute().actionGet());
    } catch (Exception e) {
        String errorMsg = String.format("Failed to get executions for event: %s", event);
        logger.error(errorMsg, e);
        throw new ApplicationException(Code.BACKEND_ERROR, errorMsg, e);
    }
}
Also used : ApplicationException(com.netflix.conductor.core.execution.ApplicationException) SearchRequestBuilder(org.elasticsearch.action.search.SearchRequestBuilder) Expression(com.netflix.conductor.dao.es5.index.query.parser.Expression) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) IndexNotFoundException(org.elasticsearch.index.IndexNotFoundException) ParserException(com.netflix.conductor.elasticsearch.query.parser.ParserException) ResourceAlreadyExistsException(org.elasticsearch.ResourceAlreadyExistsException) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) IOException(java.io.IOException)

Example 29 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchDAOV5 method updateWorkflow.

@Override
public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {
    if (keys.length != values.length) {
        throw new ApplicationException(Code.INVALID_INPUT, "Number of keys and values do not match");
    }
    long startTime = Instant.now().toEpochMilli();
    UpdateRequest request = new UpdateRequest(indexName, WORKFLOW_DOC_TYPE, workflowInstanceId);
    Map<String, Object> source = IntStream.range(0, keys.length).boxed().collect(Collectors.toMap(i -> keys[i], i -> values[i]));
    request.doc(source);
    logger.debug("Updating workflow {} in elasticsearch index: {}", workflowInstanceId, indexName);
    new RetryUtil<>().retryOnException(() -> elasticSearchClient.update(request).actionGet(), null, null, RETRY_COUNT, "Updating index for doc_type workflow", "updateWorkflow");
    long endTime = Instant.now().toEpochMilli();
    logger.debug("Time taken {} for updating workflow: {}", endTime - startTime, workflowInstanceId);
    Monitors.recordESIndexTime("update_workflow", WORKFLOW_DOC_TYPE, endTime - startTime);
    Monitors.recordWorkerQueueSize("indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size());
}
Also used : Arrays(java.util.Arrays) GetResponse(org.elasticsearch.action.get.GetResponse) SortBuilders(org.elasticsearch.search.sort.SortBuilders) SearchHits(org.elasticsearch.search.SearchHits) Date(java.util.Date) LoggerFactory(org.slf4j.LoggerFactory) Task(com.netflix.conductor.common.metadata.tasks.Task) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) StringUtils(org.apache.commons.lang3.StringUtils) WorkflowSummary(com.netflix.conductor.common.run.WorkflowSummary) PreDestroy(javax.annotation.PreDestroy) DeleteRequest(org.elasticsearch.action.delete.DeleteRequest) IndexRequest(org.elasticsearch.action.index.IndexRequest) Settings(org.elasticsearch.common.settings.Settings) TypeFactory(com.fasterxml.jackson.databind.type.TypeFactory) UpdateResponse(org.elasticsearch.action.update.UpdateResponse) Workflow(com.netflix.conductor.common.run.Workflow) IndexNotFoundException(org.elasticsearch.index.IndexNotFoundException) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) EventExecution(com.netflix.conductor.common.metadata.events.EventExecution) ZoneOffset(java.time.ZoneOffset) DeleteResponse(org.elasticsearch.action.delete.DeleteResponse) MapType(com.fasterxml.jackson.databind.type.MapType) Code(com.netflix.conductor.core.execution.ApplicationException.Code) SearchHit(org.elasticsearch.search.SearchHit) Message(com.netflix.conductor.core.events.queue.Message) SearchResult(com.netflix.conductor.common.run.SearchResult) GetRequest(org.elasticsearch.action.get.GetRequest) Trace(com.netflix.conductor.annotations.Trace) TaskExecLog(com.netflix.conductor.common.metadata.tasks.TaskExecLog) TimeZone(java.util.TimeZone) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) BulkResponse(org.elasticsearch.action.bulk.BulkResponse) FieldSortBuilder(org.elasticsearch.search.sort.FieldSortBuilder) Instant(java.time.Instant) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) Objects(java.util.Objects) IOUtils(org.apache.commons.io.IOUtils) List(java.util.List) LocalDate(java.time.LocalDate) SortOrder(org.elasticsearch.search.sort.SortOrder) ParserException(com.netflix.conductor.elasticsearch.query.parser.ParserException) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) BulkRequestBuilder(org.elasticsearch.action.bulk.BulkRequestBuilder) FetchSourceContext(org.elasticsearch.search.fetch.subphase.FetchSourceContext) IntStream(java.util.stream.IntStream) XContentFactory(org.elasticsearch.common.xcontent.XContentFactory) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) XContentType(org.elasticsearch.common.xcontent.XContentType) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) GetMappingsResponse(org.elasticsearch.action.admin.indices.mapping.get.GetMappingsResponse) SimpleDateFormat(java.text.SimpleDateFormat) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Singleton(javax.inject.Singleton) Expression(com.netflix.conductor.dao.es5.index.query.parser.Expression) ArrayList(java.util.ArrayList) ResourceAlreadyExistsException(org.elasticsearch.ResourceAlreadyExistsException) Strings(org.elasticsearch.common.Strings) Inject(javax.inject.Inject) CreateIndexRequest(org.elasticsearch.action.admin.indices.create.CreateIndexRequest) StreamSupport(java.util.stream.StreamSupport) LinkedList(java.util.LinkedList) ExecutorService(java.util.concurrent.ExecutorService) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Logger(org.slf4j.Logger) RetryUtil(com.netflix.conductor.common.utils.RetryUtil) TaskSummary(com.netflix.conductor.common.run.TaskSummary) Client(org.elasticsearch.client.Client) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DateTime(org.joda.time.DateTime) IOException(java.io.IOException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) DocWriteResponse(org.elasticsearch.action.DocWriteResponse) Monitors(com.netflix.conductor.metrics.Monitors) TimeUnit(java.util.concurrent.TimeUnit) GetIndexTemplatesResponse(org.elasticsearch.action.admin.indices.template.get.GetIndexTemplatesResponse) IndexDAO(com.netflix.conductor.dao.IndexDAO) SearchRequestBuilder(org.elasticsearch.action.search.SearchRequestBuilder) ElasticSearchConfiguration(com.netflix.conductor.elasticsearch.ElasticSearchConfiguration) InputStream(java.io.InputStream) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) UpdateRequest(org.elasticsearch.action.update.UpdateRequest) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor)

Example 30 with ApplicationException

use of com.netflix.conductor.core.execution.ApplicationException in project conductor by Netflix.

the class ElasticSearchDAOV5 method search.

private SearchResult<String> search(String indexName, String structuredQuery, int start, int size, List<String> sortOptions, String freeTextQuery, String docType) {
    try {
        QueryBuilder queryBuilder = QueryBuilders.matchAllQuery();
        if (StringUtils.isNotEmpty(structuredQuery)) {
            Expression expression = Expression.fromString(structuredQuery);
            queryBuilder = expression.getFilterBuilder();
        }
        BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder);
        QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery(freeTextQuery);
        BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery);
        final SearchRequestBuilder srb = elasticSearchClient.prepareSearch(indexName).setQuery(fq).setTypes(docType).storedFields("_id").setFrom(start).setSize(size);
        if (sortOptions != null) {
            sortOptions.forEach(sortOption -> addSortOptionToSearchRequest(srb, sortOption));
        }
        SearchResponse response = srb.get();
        LinkedList<String> result = StreamSupport.stream(response.getHits().spliterator(), false).map(SearchHit::getId).collect(Collectors.toCollection(LinkedList::new));
        long count = response.getHits().getTotalHits();
        return new SearchResult<>(count, result);
    } catch (ParserException e) {
        String errorMsg = String.format("Error performing search on index:%s with docType:%s", indexName, docType);
        logger.error(errorMsg);
        throw new ApplicationException(Code.BACKEND_ERROR, errorMsg, e);
    }
}
Also used : ParserException(com.netflix.conductor.elasticsearch.query.parser.ParserException) ApplicationException(com.netflix.conductor.core.execution.ApplicationException) SearchRequestBuilder(org.elasticsearch.action.search.SearchRequestBuilder) Expression(com.netflix.conductor.dao.es5.index.query.parser.Expression) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) SearchResult(com.netflix.conductor.common.run.SearchResult) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) SearchResponse(org.elasticsearch.action.search.SearchResponse)

Aggregations

ApplicationException (com.netflix.conductor.core.execution.ApplicationException)93 Task (com.netflix.conductor.common.metadata.tasks.Task)22 Workflow (com.netflix.conductor.common.run.Workflow)22 HashMap (java.util.HashMap)19 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)16 IOException (java.io.IOException)16 Map (java.util.Map)15 ArrayList (java.util.ArrayList)14 List (java.util.List)14 Inject (javax.inject.Inject)14 Singleton (javax.inject.Singleton)14 Logger (org.slf4j.Logger)14 LoggerFactory (org.slf4j.LoggerFactory)14 Trace (com.netflix.conductor.annotations.Trace)13 EventExecution (com.netflix.conductor.common.metadata.events.EventExecution)13 Monitors (com.netflix.conductor.metrics.Monitors)13 Collectors (java.util.stream.Collectors)13 ResultSet (com.datastax.driver.core.ResultSet)12 EventHandler (com.netflix.conductor.common.metadata.events.EventHandler)12 TimeUnit (java.util.concurrent.TimeUnit)12