Examples with TaskExecutionInfo org.apache.gobblin.rest.TaskExecutionInfo used on opensource projects

Example 6 with TaskExecutionInfo

use of org.apache.gobblin.rest.TaskExecutionInfo in project incubator-gobblin by apache.

the class TaskState method toTaskExecutionInfo.

/**
 * Convert this {@link TaskState} instance to a {@link TaskExecutionInfo} instance.
 *
 * @return a {@link TaskExecutionInfo} instance
 */
public TaskExecutionInfo toTaskExecutionInfo() {
    TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo();
    taskExecutionInfo.setJobId(this.jobId);
    taskExecutionInfo.setTaskId(this.taskId);
    if (this.startTime > 0) {
        taskExecutionInfo.setStartTime(this.startTime);
    }
    if (this.endTime > 0) {
        taskExecutionInfo.setEndTime(this.endTime);
    }
    taskExecutionInfo.setDuration(this.duration);
    taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name()));
    if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) {
        taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));
    }
    taskExecutionInfo.setHighWatermark(this.getHighWaterMark());
    // Add extract/table information
    Table table = new Table();
    Extract extract = this.getExtract();
    table.setNamespace(extract.getNamespace());
    table.setName(extract.getTable());
    if (extract.hasType()) {
        table.setType(TableTypeEnum.valueOf(extract.getType().name()));
    }
    taskExecutionInfo.setTable(table);
    // Add task metrics
    TaskMetrics taskMetrics = TaskMetrics.get(this);
    MetricArray metricArray = new MetricArray();
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getCounters().entrySet()) {
        Metric counter = new Metric();
        counter.setGroup(MetricGroup.TASK.name());
        counter.setName(entry.getKey());
        counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name()));
        counter.setValue(Long.toString(((Counter) entry.getValue()).getCount()));
        metricArray.add(counter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getMeters().entrySet()) {
        Metric meter = new Metric();
        meter.setGroup(MetricGroup.TASK.name());
        meter.setName(entry.getKey());
        meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name()));
        meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate()));
        metricArray.add(meter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getGauges().entrySet()) {
        Metric gauge = new Metric();
        gauge.setGroup(MetricGroup.TASK.name());
        gauge.setName(entry.getKey());
        gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name()));
        gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString());
        metricArray.add(gauge);
    }
    taskExecutionInfo.setMetrics(metricArray);
    // Add task properties
    Map<String, String> taskProperties = Maps.newHashMap();
    for (String name : this.getPropertyNames()) {
        String value = this.getProp(name);
        if (!Strings.isNullOrEmpty(value))
            taskProperties.put(name, value);
    }
    taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
    return taskExecutionInfo;
}

Example 7 with TaskExecutionInfo

use of org.apache.gobblin.rest.TaskExecutionInfo in project incubator-gobblin by apache.

the class JobStateTest method testToJobExecutionInfo.

@Test(dependsOnMethods = { "testSetAndGet" })
public void testToJobExecutionInfo() {
    JobExecutionInfo jobExecutionInfo = this.jobState.toJobExecutionInfo();
    Assert.assertEquals(jobExecutionInfo.getJobName(), "TestJob");
    Assert.assertEquals(jobExecutionInfo.getJobId(), "TestJob-1");
    Assert.assertEquals(jobExecutionInfo.getStartTime().longValue(), this.startTime);
    Assert.assertEquals(jobExecutionInfo.getEndTime().longValue(), this.startTime + 1000);
    Assert.assertEquals(jobExecutionInfo.getDuration().longValue(), 1000L);
    Assert.assertEquals(jobExecutionInfo.getState().name(), JobState.RunningState.COMMITTED.name());
    Assert.assertEquals(jobExecutionInfo.getLaunchedTasks().intValue(), 3);
    Assert.assertEquals(jobExecutionInfo.getCompletedTasks().intValue(), 3);
    Assert.assertEquals(jobExecutionInfo.getJobProperties().get("foo"), "bar");
    List<String> taskStateIds = Lists.newArrayList();
    for (TaskExecutionInfo taskExecutionInfo : jobExecutionInfo.getTaskExecutions()) {
        Assert.assertEquals(taskExecutionInfo.getJobId(), "TestJob-1");
        Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime);
        Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000);
        Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000);
        Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name());
        Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar");
        taskStateIds.add(taskExecutionInfo.getTaskId());
    }
    Collections.sort(taskStateIds);
    Assert.assertEquals(taskStateIds, Lists.newArrayList("TestTask-0", "TestTask-1", "TestTask-2"));
}

Example 8 with TaskExecutionInfo

use of org.apache.gobblin.rest.TaskExecutionInfo in project incubator-gobblin by apache.

the class TaskStateTest method testToTaskExecutionInfo.

@Test(dependsOnMethods = { "testSetAndGet" })
public void testToTaskExecutionInfo() {
    TaskExecutionInfo taskExecutionInfo = this.taskState.toTaskExecutionInfo();
    Assert.assertEquals(taskExecutionInfo.getJobId(), "Job-1");
    Assert.assertEquals(taskExecutionInfo.getTaskId(), "Task-1");
    Assert.assertEquals(taskExecutionInfo.getHighWatermark().longValue(), 2000L);
    Assert.assertEquals(taskExecutionInfo.getStartTime().longValue(), this.startTime);
    Assert.assertEquals(taskExecutionInfo.getEndTime().longValue(), this.startTime + 1000);
    Assert.assertEquals(taskExecutionInfo.getDuration().longValue(), 1000L);
    Assert.assertEquals(taskExecutionInfo.getState().name(), WorkUnitState.WorkingState.COMMITTED.name());
    Assert.assertEquals(taskExecutionInfo.getTaskProperties().get("foo"), "bar");
}

Example 9 with TaskExecutionInfo

use of org.apache.gobblin.rest.TaskExecutionInfo in project incubator-gobblin by apache.

the class DatabaseJobHistoryStoreV100 method resultSetToTaskExecutionInfo.

private static TaskExecutionInfo resultSetToTaskExecutionInfo(ResultSet rs) throws SQLException {
    TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo();
    taskExecutionInfo.setTaskId(rs.getString("task_id"));
    taskExecutionInfo.setJobId(rs.getString("job_id"));
    try {
        taskExecutionInfo.setStartTime(rs.getTimestamp("start_time").getTime());
    } catch (SQLException se) {
        taskExecutionInfo.setStartTime(0);
    }
    try {
        taskExecutionInfo.setEndTime(rs.getTimestamp("end_time").getTime());
    } catch (SQLException se) {
        taskExecutionInfo.setEndTime(0);
    }
    taskExecutionInfo.setDuration(rs.getLong("duration"));
    String state = rs.getString("state");
    if (!Strings.isNullOrEmpty(state)) {
        taskExecutionInfo.setState(TaskStateEnum.valueOf(state));
    }
    String failureException = rs.getString("failure_exception");
    if (!Strings.isNullOrEmpty(failureException)) {
        taskExecutionInfo.setFailureException(failureException);
    }
    taskExecutionInfo.setLowWatermark(rs.getLong("low_watermark"));
    taskExecutionInfo.setHighWatermark(rs.getLong("high_watermark"));
    Table table = new Table();
    String namespace = rs.getString("table_namespace");
    if (!Strings.isNullOrEmpty(namespace)) {
        table.setNamespace(namespace);
    }
    String name = rs.getString("table_name");
    if (!Strings.isNullOrEmpty(name)) {
        table.setName(name);
    }
    String type = rs.getString("table_type");
    if (!Strings.isNullOrEmpty(type)) {
        table.setType(TableTypeEnum.valueOf(type));
    }
    taskExecutionInfo.setTable(table);
    return taskExecutionInfo;
}

Example 10 with TaskExecutionInfo

use of org.apache.gobblin.rest.TaskExecutionInfo in project incubator-gobblin by apache.

the class DatabaseJobHistoryStoreV100 method processQueryById.

private JobExecutionInfo processQueryById(Connection connection, String jobId, JobExecutionQuery query, Filter tableFilter) throws SQLException {
    Preconditions.checkArgument(!Strings.isNullOrEmpty(jobId));
    // Query job execution information
    try (PreparedStatement jobExecutionQueryStatement = connection.prepareStatement(JOB_EXECUTION_QUERY_BY_JOB_ID_STATEMENT_TEMPLATE)) {
        jobExecutionQueryStatement.setString(1, jobId);
        try (ResultSet jobRs = jobExecutionQueryStatement.executeQuery()) {
            if (!jobRs.next()) {
                return null;
            }
            JobExecutionInfo jobExecutionInfo = resultSetToJobExecutionInfo(jobRs);
            // Query job metrics
            if (query.isIncludeJobMetrics()) {
                try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(JOB_METRIC_QUERY_STATEMENT_TEMPLATE)) {
                    jobMetricQueryStatement.setString(1, jobRs.getString(2));
                    try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) {
                        MetricArray jobMetrics = new MetricArray();
                        while (jobMetricRs.next()) {
                            jobMetrics.add(resultSetToMetric(jobMetricRs));
                        }
                        // Add job metrics
                        jobExecutionInfo.setMetrics(jobMetrics);
                    }
                }
            }
            // Query job properties
            Set<String> requestedJobPropertyKeys = null;
            if (query.hasJobProperties()) {
                requestedJobPropertyKeys = new HashSet<>(Arrays.asList(query.getJobProperties().split(",")));
            }
            try (PreparedStatement jobPropertiesQueryStatement = connection.prepareStatement(JOB_PROPERTY_QUERY_STATEMENT_TEMPLATE)) {
                jobPropertiesQueryStatement.setString(1, jobExecutionInfo.getJobId());
                try (ResultSet jobPropertiesRs = jobPropertiesQueryStatement.executeQuery()) {
                    Map<String, String> jobProperties = Maps.newHashMap();
                    while (jobPropertiesRs.next()) {
                        Map.Entry<String, String> property = resultSetToProperty(jobPropertiesRs);
                        if (requestedJobPropertyKeys == null || requestedJobPropertyKeys.contains(property.getKey())) {
                            jobProperties.put(property.getKey(), property.getValue());
                        }
                    }
                    // Add job properties
                    jobExecutionInfo.setJobProperties(new StringMap(jobProperties));
                }
            }
            // Query task execution information
            if (query.isIncludeTaskExecutions()) {
                TaskExecutionInfoArray taskExecutionInfos = new TaskExecutionInfoArray();
                String taskExecutionQuery = TASK_EXECUTION_QUERY_STATEMENT_TEMPLATE;
                // Add table filter if applicable
                if (tableFilter.isPresent()) {
                    taskExecutionQuery += " AND " + tableFilter;
                }
                try (PreparedStatement taskExecutionQueryStatement = connection.prepareStatement(taskExecutionQuery)) {
                    taskExecutionQueryStatement.setString(1, jobId);
                    if (tableFilter.isPresent()) {
                        tableFilter.addParameters(taskExecutionQueryStatement, 2);
                    }
                    try (ResultSet taskRs = taskExecutionQueryStatement.executeQuery()) {
                        while (taskRs.next()) {
                            TaskExecutionInfo taskExecutionInfo = resultSetToTaskExecutionInfo(taskRs);
                            // Query task metrics for each task execution record
                            if (query.isIncludeTaskMetrics()) {
                                try (PreparedStatement taskMetricQueryStatement = connection.prepareStatement(TASK_METRIC_QUERY_STATEMENT_TEMPLATE)) {
                                    taskMetricQueryStatement.setString(1, taskExecutionInfo.getTaskId());
                                    try (ResultSet taskMetricRs = taskMetricQueryStatement.executeQuery()) {
                                        MetricArray taskMetrics = new MetricArray();
                                        while (taskMetricRs.next()) {
                                            taskMetrics.add(resultSetToMetric(taskMetricRs));
                                        }
                                        // Add task metrics
                                        taskExecutionInfo.setMetrics(taskMetrics);
                                    }
                                }
                            }
                            taskExecutionInfos.add(taskExecutionInfo);
                            // Query task properties
                            Set<String> queryTaskPropertyKeys = null;
                            if (query.hasTaskProperties()) {
                                queryTaskPropertyKeys = new HashSet<>(Arrays.asList(query.getTaskProperties().split(",")));
                            }
                            try (PreparedStatement taskPropertiesQueryStatement = connection.prepareStatement(TASK_PROPERTY_QUERY_STATEMENT_TEMPLATE)) {
                                taskPropertiesQueryStatement.setString(1, taskExecutionInfo.getTaskId());
                                try (ResultSet taskPropertiesRs = taskPropertiesQueryStatement.executeQuery()) {
                                    Map<String, String> taskProperties = Maps.newHashMap();
                                    while (taskPropertiesRs.next()) {
                                        Map.Entry<String, String> property = resultSetToProperty(taskPropertiesRs);
                                        if (queryTaskPropertyKeys == null || queryTaskPropertyKeys.contains(property.getKey())) {
                                            taskProperties.put(property.getKey(), property.getValue());
                                        }
                                    }
                                    // Add job properties
                                    taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
                                }
                            }
                        // Add task properties
                        }
                        // Add task execution information
                        jobExecutionInfo.setTaskExecutions(taskExecutionInfos);
                    }
                }
            }
            return jobExecutionInfo;
        }
    }
}