Examples with TaskMetrics org.apache.gobblin.runtime.util.TaskMetrics used on opensource projects

Example 1 with TaskMetrics

use of org.apache.gobblin.runtime.util.TaskMetrics in project incubator-gobblin by apache.

the class TaskState method toTaskExecutionInfo.

/**
 * Convert this {@link TaskState} instance to a {@link TaskExecutionInfo} instance.
 *
 * @return a {@link TaskExecutionInfo} instance
 */
public TaskExecutionInfo toTaskExecutionInfo() {
    TaskExecutionInfo taskExecutionInfo = new TaskExecutionInfo();
    taskExecutionInfo.setJobId(this.jobId);
    taskExecutionInfo.setTaskId(this.taskId);
    if (this.startTime > 0) {
        taskExecutionInfo.setStartTime(this.startTime);
    }
    if (this.endTime > 0) {
        taskExecutionInfo.setEndTime(this.endTime);
    }
    taskExecutionInfo.setDuration(this.duration);
    taskExecutionInfo.setState(TaskStateEnum.valueOf(getWorkingState().name()));
    if (this.contains(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY)) {
        taskExecutionInfo.setFailureException(this.getProp(ConfigurationKeys.TASK_FAILURE_EXCEPTION_KEY));
    }
    taskExecutionInfo.setHighWatermark(this.getHighWaterMark());
    // Add extract/table information
    Table table = new Table();
    Extract extract = this.getExtract();
    table.setNamespace(extract.getNamespace());
    table.setName(extract.getTable());
    if (extract.hasType()) {
        table.setType(TableTypeEnum.valueOf(extract.getType().name()));
    }
    taskExecutionInfo.setTable(table);
    // Add task metrics
    TaskMetrics taskMetrics = TaskMetrics.get(this);
    MetricArray metricArray = new MetricArray();
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getCounters().entrySet()) {
        Metric counter = new Metric();
        counter.setGroup(MetricGroup.TASK.name());
        counter.setName(entry.getKey());
        counter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.COUNTER.name()));
        counter.setValue(Long.toString(((Counter) entry.getValue()).getCount()));
        metricArray.add(counter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getMeters().entrySet()) {
        Metric meter = new Metric();
        meter.setGroup(MetricGroup.TASK.name());
        meter.setName(entry.getKey());
        meter.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.METER.name()));
        meter.setValue(Double.toString(((Meter) entry.getValue()).getMeanRate()));
        metricArray.add(meter);
    }
    for (Map.Entry<String, ? extends com.codahale.metrics.Metric> entry : taskMetrics.getMetricContext().getGauges().entrySet()) {
        Metric gauge = new Metric();
        gauge.setGroup(MetricGroup.TASK.name());
        gauge.setName(entry.getKey());
        gauge.setType(MetricTypeEnum.valueOf(GobblinMetrics.MetricType.GAUGE.name()));
        gauge.setValue(((Gauge<?>) entry.getValue()).getValue().toString());
        metricArray.add(gauge);
    }
    taskExecutionInfo.setMetrics(metricArray);
    // Add task properties
    Map<String, String> taskProperties = Maps.newHashMap();
    for (String name : this.getPropertyNames()) {
        String value = this.getProp(name);
        if (!Strings.isNullOrEmpty(value))
            taskProperties.put(name, value);
    }
    taskExecutionInfo.setTaskProperties(new StringMap(taskProperties));
    return taskExecutionInfo;
}

Example 2 with TaskMetrics

use of org.apache.gobblin.runtime.util.TaskMetrics in project incubator-gobblin by apache.

the class TaskState method adjustJobMetricsOnRetry.

/**
 * Adjust job-level metrics when the task gets retried.
 *
 * @param branches number of forked branches
 */
public void adjustJobMetricsOnRetry(int branches) {
    TaskMetrics metrics = TaskMetrics.get(this);
    for (int i = 0; i < branches; i++) {
        String forkBranchId = ForkOperatorUtils.getForkId(this.taskId, i);
        long recordsWritten = metrics.getCounter(MetricGroup.TASK.name(), forkBranchId, RECORDS).getCount();
        long bytesWritten = metrics.getCounter(MetricGroup.TASK.name(), forkBranchId, BYTES).getCount();
        metrics.getCounter(MetricGroup.JOB.name(), this.jobId, RECORDS).dec(recordsWritten);
        metrics.getCounter(MetricGroup.JOB.name(), this.jobId, BYTES).dec(bytesWritten);
    }
}

Example 3 with TaskMetrics

use of org.apache.gobblin.runtime.util.TaskMetrics in project incubator-gobblin by apache.

the class TaskState method updateByteMetrics.

/**
 * Collect byte-level metrics.
 *
 * @param bytesWritten number of bytes written by the writer
 * @param branchIndex fork branch index
 *
 * @deprecated see {@link org.apache.gobblin.instrumented.writer.InstrumentedDataWriterBase}.
 */
public synchronized void updateByteMetrics(long bytesWritten, int branchIndex) {
    TaskMetrics metrics = TaskMetrics.get(this);
    String forkBranchId = TaskMetrics.taskInstanceRemoved(this.taskId);
    Counter taskByteCounter = metrics.getCounter(MetricGroup.TASK.name(), forkBranchId, BYTES);
    long inc = bytesWritten - taskByteCounter.getCount();
    taskByteCounter.inc(inc);
    metrics.getMeter(MetricGroup.TASK.name(), forkBranchId, BYTES_PER_SECOND).mark(inc);
    metrics.getCounter(MetricGroup.JOB.name(), this.jobId, BYTES).inc(inc);
    metrics.getMeter(MetricGroup.JOB.name(), this.jobId, BYTES_PER_SECOND).mark(inc);
}

Example 4 with TaskMetrics

use of org.apache.gobblin.runtime.util.TaskMetrics in project incubator-gobblin by apache.

the class TaskState method updateRecordMetrics.

/**
 * Update record-level metrics.
 *
 * @param recordsWritten number of records written by the writer
 * @param branchIndex fork branch index
 *
 * @deprecated see {@link org.apache.gobblin.instrumented.writer.InstrumentedDataWriterBase}.
 */
public synchronized void updateRecordMetrics(long recordsWritten, int branchIndex) {
    TaskMetrics metrics = TaskMetrics.get(this);
    // chopping branch index from metric name
    // String forkBranchId = ForkOperatorUtils.getForkId(this.taskId, branchIndex);
    String forkBranchId = TaskMetrics.taskInstanceRemoved(this.taskId);
    Counter taskRecordCounter = metrics.getCounter(MetricGroup.TASK.name(), forkBranchId, RECORDS);
    long inc = recordsWritten - taskRecordCounter.getCount();
    taskRecordCounter.inc(inc);
    metrics.getMeter(MetricGroup.TASK.name(), forkBranchId, RECORDS_PER_SECOND).mark(inc);
    metrics.getCounter(MetricGroup.JOB.name(), this.jobId, RECORDS).inc(inc);
    metrics.getMeter(MetricGroup.JOB.name(), this.jobId, RECORDS_PER_SECOND).mark(inc);
}