Examples with TaskExecLog com.netflix.conductor.common.metadata.tasks.TaskExecLog used on opensource projects

Example 1 with TaskExecLog

use of com.netflix.conductor.common.metadata.tasks.TaskExecLog in project conductor by Netflix.

the class TestElasticSearchRestDAOV7 method createLog.

private TaskExecLog createLog(String taskId, String log) {
    TaskExecLog taskExecLog = new TaskExecLog(log);
    taskExecLog.setTaskId(taskId);
    return taskExecLog;
}

Example 2 with TaskExecLog

use of com.netflix.conductor.common.metadata.tasks.TaskExecLog in project conductor by Netflix.

the class ElasticSearchRestDAOV5 method getTaskExecutionLogs.

@Override
public List<TaskExecLog> getTaskExecutionLogs(String taskId) {
    try {
        // Build Query
        Expression expression = Expression.fromString("taskId='" + taskId + "'");
        QueryBuilder queryBuilder = expression.getFilterBuilder();
        BoolQueryBuilder filterQuery = QueryBuilders.boolQuery().must(queryBuilder);
        QueryStringQueryBuilder stringQuery = QueryBuilders.queryStringQuery("*");
        BoolQueryBuilder fq = QueryBuilders.boolQuery().must(stringQuery).must(filterQuery);
        // Create the searchObjectIdsViaExpression source
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        searchSourceBuilder.query(fq);
        searchSourceBuilder.sort(new FieldSortBuilder("createdTime").order(SortOrder.ASC));
        searchSourceBuilder.size(config.getElasticSearchTasklogLimit());
        // Generate the actual request to send to ES.
        SearchRequest searchRequest = new SearchRequest(logIndexPrefix + "*");
        searchRequest.types(LOG_DOC_TYPE);
        searchRequest.source(searchSourceBuilder);
        SearchResponse response = elasticSearchClient.search(searchRequest);
        SearchHit[] hits = response.getHits().getHits();
        List<TaskExecLog> logs = new ArrayList<>(hits.length);
        for (SearchHit hit : hits) {
            String source = hit.getSourceAsString();
            TaskExecLog tel = objectMapper.readValue(source, TaskExecLog.class);
            logs.add(tel);
        }
        return logs;
    } catch (Exception e) {
        logger.error("Failed to get task execution logs for task: {}", taskId, e);
    }
    return null;
}

Example 3 with TaskExecLog

use of com.netflix.conductor.common.metadata.tasks.TaskExecLog in project conductor by Netflix.

the class ElasticSearchDAOV6 method addTaskExecutionLogs.

@Override
public void addTaskExecutionLogs(List<TaskExecLog> taskExecLogs) {
    if (taskExecLogs.isEmpty()) {
        return;
    }
    try {
        long startTime = Instant.now().toEpochMilli();
        BulkRequestBuilderWrapper bulkRequestBuilder = new BulkRequestBuilderWrapper(elasticSearchClient.prepareBulk());
        for (TaskExecLog log : taskExecLogs) {
            String docType = StringUtils.isBlank(docTypeOverride) ? LOG_DOC_TYPE : docTypeOverride;
            IndexRequest request = new IndexRequest(logIndexName, docType);
            request.source(objectMapper.writeValueAsBytes(log), XContentType.JSON);
            bulkRequestBuilder.add(request);
        }
        new RetryUtil<BulkResponse>().retryOnException(() -> bulkRequestBuilder.execute().actionGet(5, TimeUnit.SECONDS), null, BulkResponse::hasFailures, RETRY_COUNT, "Indexing task execution logs", "addTaskExecutionLogs");
        long endTime = Instant.now().toEpochMilli();
        LOGGER.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime);
        Monitors.recordESIndexTime("index_task_execution_logs", LOG_DOC_TYPE, endTime - startTime);
        Monitors.recordWorkerQueueSize("logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size());
    } catch (Exception e) {
        List<String> taskIds = taskExecLogs.stream().map(TaskExecLog::getTaskId).collect(Collectors.toList());
        LOGGER.error("Failed to index task execution logs for tasks: {}", taskIds, e);
    }
}

Example 4 with TaskExecLog

use of com.netflix.conductor.common.metadata.tasks.TaskExecLog in project conductor by Netflix.

the class TestElasticSearchRestDAOV5 method taskExecutionLogs.

@Test
public void taskExecutionLogs() throws Exception {
    TaskExecLog taskExecLog1 = new TaskExecLog();
    taskExecLog1.setTaskId("some-task-id");
    long createdTime1 = LocalDateTime.of(2018, 11, 01, 06, 33, 22).toEpochSecond(ZoneOffset.UTC);
    taskExecLog1.setCreatedTime(createdTime1);
    taskExecLog1.setLog("some-log");
    TaskExecLog taskExecLog2 = new TaskExecLog();
    taskExecLog2.setTaskId("some-task-id");
    long createdTime2 = LocalDateTime.of(2018, 11, 01, 06, 33, 22).toEpochSecond(ZoneOffset.UTC);
    taskExecLog2.setCreatedTime(createdTime2);
    taskExecLog2.setLog("some-log");
    List<TaskExecLog> logsToAdd = Arrays.asList(taskExecLog1, taskExecLog2);
    indexDAO.addTaskExecutionLogs(logsToAdd);
    await().atMost(5, TimeUnit.SECONDS).untilAsserted(() -> {
        List<TaskExecLog> taskExecutionLogs = indexDAO.getTaskExecutionLogs("some-task-id");
        assertEquals(2, taskExecutionLogs.size());
    });
}

Example 5 with TaskExecLog

use of com.netflix.conductor.common.metadata.tasks.TaskExecLog in project conductor by Netflix.

the class ElasticSearchRestDAOV7 method addTaskExecutionLogs.

@Override
public void addTaskExecutionLogs(List<TaskExecLog> taskExecLogs) {
    if (taskExecLogs.isEmpty()) {
        return;
    }
    long startTime = Instant.now().toEpochMilli();
    BulkRequest bulkRequest = new BulkRequest();
    for (TaskExecLog log : taskExecLogs) {
        byte[] docBytes;
        try {
            docBytes = objectMapper.writeValueAsBytes(log);
        } catch (JsonProcessingException e) {
            logger.error("Failed to convert task log to JSON for task {}", log.getTaskId());
            continue;
        }
        IndexRequest request = new IndexRequest(logIndexName);
        request.source(docBytes, XContentType.JSON);
        bulkRequest.add(request);
    }
    try {
        new RetryUtil<BulkResponse>().retryOnException(() -> {
            try {
                return elasticSearchClient.bulk(bulkRequest, RequestOptions.DEFAULT);
            } catch (IOException e) {
                throw new RuntimeException(e);
            }
        }, null, BulkResponse::hasFailures, RETRY_COUNT, "Indexing task execution logs", "addTaskExecutionLogs");
        long endTime = Instant.now().toEpochMilli();
        logger.debug("Time taken {} for indexing taskExecutionLogs", endTime - startTime);
        Monitors.recordESIndexTime("index_task_execution_logs", LOG_DOC_TYPE, endTime - startTime);
        Monitors.recordWorkerQueueSize("logQueue", ((ThreadPoolExecutor) logExecutorService).getQueue().size());
    } catch (Exception e) {
        List<String> taskIds = taskExecLogs.stream().map(TaskExecLog::getTaskId).collect(Collectors.toList());
        logger.error("Failed to index task execution logs for tasks: {}", taskIds, e);
    }
}