Examples with MetricArray org.apache.gobblin.rest.MetricArray used on opensource projects

Example 1 with MetricArray

use of org.apache.gobblin.rest.MetricArray in project incubator-gobblin by apache.

the class DatabaseJobHistoryStoreV101 method addMetricsToJobExecutions.

private void addMetricsToJobExecutions(Connection connection, JobExecutionQuery query, Map<String, JobExecutionInfo> jobExecutionInfos) throws SQLException {
    if (query.isIncludeJobMetrics() && jobExecutionInfos.size() > 0) {
        String template = String.format(JOB_METRIC_QUERY_STATEMENT_TEMPLATE, getInPredicate(jobExecutionInfos.size()));
        int index = 1;
        try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(template)) {
            for (String jobId : jobExecutionInfos.keySet()) {
                jobMetricQueryStatement.setString(index++, jobId);
            }
            try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) {
                while (jobMetricRs.next()) {
                    String jobId = jobMetricRs.getString("job_id");
                    JobExecutionInfo jobExecutionInfo = jobExecutionInfos.get(jobId);
                    MetricArray metricArray = jobExecutionInfo.getMetrics(GetMode.NULL);
                    if (metricArray == null) {
                        metricArray = new MetricArray();
                        jobExecutionInfo.setMetrics(metricArray);
                    }
                    metricArray.add(resultSetToMetric(jobMetricRs));
                }
            }
        }
    }
}

Example 2 with MetricArray

use of org.apache.gobblin.rest.MetricArray in project incubator-gobblin by apache.

the class DatabaseJobHistoryStoreTest method create.

private JobExecutionInfo create(int index, boolean differentTableType) {
    JobExecutionInfo jobExecutionInfo = new JobExecutionInfo();
    jobExecutionInfo.setJobName("TestJob" + index);
    jobExecutionInfo.setJobId(jobExecutionInfo.getJobName() + "_" + System.currentTimeMillis());
    jobExecutionInfo.setStartTime(System.currentTimeMillis());
    jobExecutionInfo.setState(JobStateEnum.PENDING);
    jobExecutionInfo.setLaunchedTasks(2);
    jobExecutionInfo.setCompletedTasks(0);
    jobExecutionInfo.setLauncherType(LauncherTypeEnum.LOCAL);
    jobExecutionInfo.setTrackingUrl("localhost");
    MetricArray jobMetrics = new MetricArray();
    Metric jobMetric1 = new Metric();
    jobMetric1.setGroup("JOB");
    jobMetric1.setName("jm1");
    jobMetric1.setType(MetricTypeEnum.COUNTER);
    jobMetric1.setValue("100");
    jobMetrics.add(jobMetric1);
    jobExecutionInfo.setMetrics(jobMetrics);
    Map<String, String> jobProperties = Maps.newHashMap();
    jobProperties.put("k" + index, "v" + index);
    jobExecutionInfo.setJobProperties(new StringMap(jobProperties));
    TaskExecutionInfoArray taskExecutionInfos = new TaskExecutionInfoArray();
    TaskExecutionInfo taskExecutionInfo1 = new TaskExecutionInfo();
    taskExecutionInfo1.setJobId(jobExecutionInfo.getJobId());
    taskExecutionInfo1.setTaskId(jobExecutionInfo.getJobId() + "_0");
    taskExecutionInfo1.setStartTime(System.currentTimeMillis());
    taskExecutionInfo1.setState(TaskStateEnum.PENDING);
    taskExecutionInfo1.setLowWatermark(0L);
    taskExecutionInfo1.setHighWatermark(1000L);
    Table table1 = new Table();
    table1.setNamespace("Test");
    table1.setName("Test1");
    table1.setType(TableTypeEnum.SNAPSHOT_ONLY);
    taskExecutionInfo1.setTable(table1);
    MetricArray taskMetrics1 = new MetricArray();
    Metric taskMetric1 = new Metric();
    taskMetric1.setGroup("TASK");
    taskMetric1.setName("tm1");
    taskMetric1.setType(MetricTypeEnum.COUNTER);
    taskMetric1.setValue("100");
    taskMetrics1.add(taskMetric1);
    taskExecutionInfo1.setMetrics(taskMetrics1);
    Map<String, String> taskProperties1 = Maps.newHashMap();
    taskProperties1.put("k1" + index, "v1" + index);
    taskExecutionInfo1.setTaskProperties(new StringMap(taskProperties1));
    taskExecutionInfos.add(taskExecutionInfo1);
    TaskExecutionInfo taskExecutionInfo2 = new TaskExecutionInfo();
    taskExecutionInfo2.setJobId(jobExecutionInfo.getJobId());
    taskExecutionInfo2.setTaskId(jobExecutionInfo.getJobId() + "_1");
    taskExecutionInfo2.setStartTime(System.currentTimeMillis());
    taskExecutionInfo2.setState(TaskStateEnum.PENDING);
    taskExecutionInfo2.setLowWatermark(0L);
    taskExecutionInfo2.setHighWatermark(2000L);
    Table table2 = new Table();
    table2.setNamespace("Test");
    table2.setName("Test2");
    table2.setType(differentTableType ? TableTypeEnum.SNAPSHOT_APPEND : TableTypeEnum.SNAPSHOT_ONLY);
    taskExecutionInfo2.setTable(table2);
    MetricArray taskMetrics2 = new MetricArray();
    Metric taskMetric2 = new Metric();
    taskMetric2.setGroup("TASK");
    taskMetric2.setName("tm2");
    taskMetric2.setType(MetricTypeEnum.COUNTER);
    taskMetric2.setValue("100");
    taskMetrics2.add(taskMetric2);
    taskExecutionInfo2.setMetrics(taskMetrics2);
    Map<String, String> taskProperties2 = Maps.newHashMap();
    taskProperties2.put("k2" + index, "v2" + index);
    taskExecutionInfo2.setTaskProperties(new StringMap(taskProperties2));
    taskExecutionInfos.add(taskExecutionInfo2);
    jobExecutionInfo.setTaskExecutions(taskExecutionInfos);
    this.expectedJobExecutionInfos.add(jobExecutionInfo);
    return jobExecutionInfo;
}

Example 3 with MetricArray

use of org.apache.gobblin.rest.MetricArray in project incubator-gobblin by apache.

the class TaskState method toTaskExecutionInfo.

/**
 * Convert this {@link TaskState} instance to a {@link TaskExecutionInfo} instance.
 *
 * @return a {@link TaskExecutionInfo} instance
 */
public TaskExecutionInfo toTaskExecutionInfo() {
    TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo();
    taskExecutionInfo.setJobId(this.jobId);
    taskExecutionInfo.setTaskId(this.taskId);
    if (this.startTime > 0) {
        taskExecutionInfo.setStartTime(this.startTime);
    }
    if (this.endTime > 0) {
        taskExecutionInfo.setEndTime(this.endTime);
    }
    taskExecutionInfo.setDuration(this.duration);
    taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name()));
    if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) {
        taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));
    }
    taskExecutionInfo.setHighWatermark(this.getHighWaterMark());
    // Add extract/table information
    Table table = new Table();
    Extract extract = this.getExtract();
    table.setNamespace(extract.getNamespace());
    table.setName(extract.getTable());
    if (extract.hasType()) {
        table.setType(TableTypeEnum.valueOf(extract.getType().name()));
    }
    taskExecutionInfo.setTable(table);
    // Add task metrics
    TaskMetrics taskMetrics = TaskMetrics.get(this);
    MetricArray metricArray = new MetricArray();
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getCounters().entrySet()) {
        Metric counter = new Metric();
        counter.setGroup(MetricGroup.TASK.name());
        counter.setName(entry.getKey());
        counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name()));
        counter.setValue(Long.toString(((Counter) entry.getValue()).getCount()));
        metricArray.add(counter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getMeters().entrySet()) {
        Metric meter = new Metric();
        meter.setGroup(MetricGroup.TASK.name());
        meter.setName(entry.getKey());
        meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name()));
        meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate()));
        metricArray.add(meter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getGauges().entrySet()) {
        Metric gauge = new Metric();
        gauge.setGroup(MetricGroup.TASK.name());
        gauge.setName(entry.getKey());
        gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name()));
        gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString());
        metricArray.add(gauge);
    }
    taskExecutionInfo.setMetrics(metricArray);
    // Add task properties
    Map<String, String> taskProperties = Maps.newHashMap();
    for (String name : this.getPropertyNames()) {
        String value = this.getProp(name);
        if (!Strings.isNullOrEmpty(value))
            taskProperties.put(name, value);
    }
    taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
    return taskExecutionInfo;
}

Example 4 with MetricArray

use of org.apache.gobblin.rest.MetricArray in project incubator-gobblin by apache.

the class JobState method toJobExecutionInfo.

/**
 * Convert this {@link JobState} instance to a {@link JobExecutionInfo} instance.
 *
 * @return a {@link JobExecutionInfo} instance
 */
public JobExecutionInfo toJobExecutionInfo() {
    JobExecutionInfo jobExecutionInfo = new JobExecutionInfo();
    jobExecutionInfo.setJobName(this.jobName);
    jobExecutionInfo.setJobId(this.jobId);
    if (this.startTime > 0) {
        jobExecutionInfo.setStartTime(this.startTime);
    }
    if (this.endTime > 0) {
        jobExecutionInfo.setEndTime(this.endTime);
    }
    jobExecutionInfo.setDuration(this.duration);
    jobExecutionInfo.setState(JobStateEnum.valueOf(this.state.name()));
    jobExecutionInfo.setLaunchedTasks(this.taskCount);
    jobExecutionInfo.setCompletedTasks(this.getCompletedTasks());
    jobExecutionInfo.setLauncherType(getLauncherType());
    if (getTrackingURL().isPresent()) {
        jobExecutionInfo.setTrackingUrl(getTrackingURL().get());
    }
    // Add task execution information
    TaskExecutionInfoArray taskExecutionInfos = new TaskExecutionInfoArray();
    for (TaskState taskState : this.getTaskStates()) {
        taskExecutionInfos.add(taskState.toTaskExecutionInfo());
    }
    jobExecutionInfo.setTaskExecutions(taskExecutionInfos);
    // Add job metrics
    JobMetrics jobMetrics = JobMetrics.get(this);
    MetricArray metricArray = new MetricArray();
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : jobMetrics.getMetricContext().getCounters().entrySet()) {
        Metric counter = new Metric();
        counter.setGroup(MetricGroup.JOB.name());
        counter.setName(entry.getKey());
        counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name()));
        counter.setValue(Long.toString(((Counter) entry.getValue()).getCount()));
        metricArray.add(counter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : jobMetrics.getMetricContext().getMeters().entrySet()) {
        Metric meter = new Metric();
        meter.setGroup(MetricGroup.JOB.name());
        meter.setName(entry.getKey());
        meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name()));
        meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate()));
        metricArray.add(meter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : jobMetrics.getMetricContext().getGauges().entrySet()) {
        Metric gauge = new Metric();
        gauge.setGroup(MetricGroup.JOB.name());
        gauge.setName(entry.getKey());
        gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name()));
        gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString());
        metricArray.add(gauge);
    }
    jobExecutionInfo.setMetrics(metricArray);
    // Add job properties
    Map<String, String> jobProperties = Maps.newHashMap();
    for (String name : this.getPropertyNames()) {
        String value = this.getProp(name);
        if (!Strings.isNullOrEmpty(value)) {
            jobProperties.put(name, value);
        }
    }
    jobExecutionInfo.setJobProperties(new StringMap(jobProperties));
    return jobExecutionInfo;
}

Example 5 with MetricArray

use of org.apache.gobblin.rest.MetricArray in project incubator-gobblin by apache.

the class DatabaseJobHistoryStoreV100 method processQueryById.

private JobExecutionInfo processQueryById(Connection connection, String jobId, JobExecutionQuery query, Filter tableFilter) throws SQLException {
    Preconditions.checkArgument(!Strings.isNullOrEmpty(jobId));
    // Query job execution information
    try (PreparedStatement jobExecutionQueryStatement = connection.prepareStatement(JOB_EXECUTION_QUERY_BY_JOB_ID_STATEMENT_TEMPLATE)) {
        jobExecutionQueryStatement.setString(1, jobId);
        try (ResultSet jobRs = jobExecutionQueryStatement.executeQuery()) {
            if (!jobRs.next()) {
                return null;
            }
            JobExecutionInfo jobExecutionInfo = resultSetToJobExecutionInfo(jobRs);
            // Query job metrics
            if (query.isIncludeJobMetrics()) {
                try (PreparedStatement jobMetricQueryStatement = connection.prepareStatement(JOB_METRIC_QUERY_STATEMENT_TEMPLATE)) {
                    jobMetricQueryStatement.setString(1, jobRs.getString(2));
                    try (ResultSet jobMetricRs = jobMetricQueryStatement.executeQuery()) {
                        MetricArray jobMetrics = new MetricArray();
                        while (jobMetricRs.next()) {
                            jobMetrics.add(resultSetToMetric(jobMetricRs));
                        }
                        // Add job metrics
                        jobExecutionInfo.setMetrics(jobMetrics);
                    }
                }
            }
            // Query job properties
            Set<String> requestedJobPropertyKeys = null;
            if (query.hasJobProperties()) {
                requestedJobPropertyKeys = new HashSet<>(Arrays.asList(query.getJobProperties().split(",")));
            }
            try (PreparedStatement jobPropertiesQueryStatement = connection.prepareStatement(JOB_PROPERTY_QUERY_STATEMENT_TEMPLATE)) {
                jobPropertiesQueryStatement.setString(1, jobExecutionInfo.getJobId());
                try (ResultSet jobPropertiesRs = jobPropertiesQueryStatement.executeQuery()) {
                    Map<String, String> jobProperties = Maps.newHashMap();
                    while (jobPropertiesRs.next()) {
                        Map.Entry<String, String> property = resultSetToProperty(jobPropertiesRs);
                        if (requestedJobPropertyKeys == null || requestedJobPropertyKeys.contains(property.getKey())) {
                            jobProperties.put(property.getKey(), property.getValue());
                        }
                    }
                    // Add job properties
                    jobExecutionInfo.setJobProperties(new StringMap(jobProperties));
                }
            }
            // Query task execution information
            if (query.isIncludeTaskExecutions()) {
                TaskExecutionInfoArray taskExecutionInfos = new TaskExecutionInfoArray();
                String taskExecutionQuery = TASK_EXECUTION_QUERY_STATEMENT_TEMPLATE;
                // Add table filter if applicable
                if (tableFilter.isPresent()) {
                    taskExecutionQuery += " AND " + tableFilter;
                }
                try (PreparedStatement taskExecutionQueryStatement = connection.prepareStatement(taskExecutionQuery)) {
                    taskExecutionQueryStatement.setString(1, jobId);
                    if (tableFilter.isPresent()) {
                        tableFilter.addParameters(taskExecutionQueryStatement, 2);
                    }
                    try (ResultSet taskRs = taskExecutionQueryStatement.executeQuery()) {
                        while (taskRs.next()) {
                            TaskExecutionInfo taskExecutionInfo = resultSetToTaskExecutionInfo(taskRs);
                            // Query task metrics for each task execution record
                            if (query.isIncludeTaskMetrics()) {
                                try (PreparedStatement taskMetricQueryStatement = connection.prepareStatement(TASK_METRIC_QUERY_STATEMENT_TEMPLATE)) {
                                    taskMetricQueryStatement.setString(1, taskExecutionInfo.getTaskId());
                                    try (ResultSet taskMetricRs = taskMetricQueryStatement.executeQuery()) {
                                        MetricArray taskMetrics = new MetricArray();
                                        while (taskMetricRs.next()) {
                                            taskMetrics.add(resultSetToMetric(taskMetricRs));
                                        }
                                        // Add task metrics
                                        taskExecutionInfo.setMetrics(taskMetrics);
                                    }
                                }
                            }
                            taskExecutionInfos.add(taskExecutionInfo);
                            // Query task properties
                            Set<String> queryTaskPropertyKeys = null;
                            if (query.hasTaskProperties()) {
                                queryTaskPropertyKeys = new HashSet<>(Arrays.asList(query.getTaskProperties().split(",")));
                            }
                            try (PreparedStatement taskPropertiesQueryStatement = connection.prepareStatement(TASK_PROPERTY_QUERY_STATEMENT_TEMPLATE)) {
                                taskPropertiesQueryStatement.setString(1, taskExecutionInfo.getTaskId());
                                try (ResultSet taskPropertiesRs = taskPropertiesQueryStatement.executeQuery()) {
                                    Map<String, String> taskProperties = Maps.newHashMap();
                                    while (taskPropertiesRs.next()) {
                                        Map.Entry<String, String> property = resultSetToProperty(taskPropertiesRs);
                                        if (queryTaskPropertyKeys == null || queryTaskPropertyKeys.contains(property.getKey())) {
                                            taskProperties.put(property.getKey(), property.getValue());
                                        }
                                    }
                                    // Add job properties
                                    taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
                                }
                            }
                        // Add task properties
                        }
                        // Add task execution information
                        jobExecutionInfo.setTaskExecutions(taskExecutionInfos);
                    }
                }
            }
            return jobExecutionInfo;
        }
    }
}