Examples with JobState org.apache.gobblin.runtime.JobState used on opensource projects

Example 6 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class HiveMaterializerTest method getTaskContextForRun.

private TaskContext getTaskContextForRun(WorkUnit workUnit) {
    workUnit.setProp(ConfigurationKeys.JOB_ID_KEY, "job123");
    workUnit.setProp(ConfigurationKeys.TASK_ID_KEY, "task123");
    workUnit.setProp(HiveConverterUtils.HIVE_DATASET_DESTINATION_SKIP_SETGROUP, Boolean.toString(true));
    HiveTask.disableHiveWatermarker(workUnit);
    JobState jobState = new JobState("job", "job123");
    return new TaskContext(new WorkUnitState(workUnit, jobState));
}

Example 7 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class TaskMetricsTest method testTaskGetMetrics.

@Test
public void testTaskGetMetrics() {
    String jobId = "job_456";
    String taskId = "task_456";
    String jobName = "jobName";
    JobState jobState = new JobState(jobName, jobId);
    JobMetrics jobMetrics = JobMetrics.get(jobState);
    State props = new State();
    props.setProp(ConfigurationKeys.JOB_ID_KEY, jobId);
    props.setProp(ConfigurationKeys.TASK_ID_KEY, taskId);
    SourceState sourceState = new SourceState(props, new ArrayList<WorkUnitState>());
    WorkUnit workUnit = new WorkUnit(sourceState, null);
    WorkUnitState workUnitState = new WorkUnitState(workUnit);
    TaskState taskState = new TaskState(workUnitState);
    TaskMetrics taskMetrics = new TaskMetrics(taskState);
    Assert.assertNotNull(taskMetrics.getMetricContext());
    Assert.assertTrue(taskMetrics.getMetricContext().getParent().isPresent());
    Assert.assertEquals(taskMetrics.getMetricContext().getParent().get(), jobMetrics.getMetricContext());
}

Example 8 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class JobMetricsTest method testJobMetricsGet.

@Test
public void testJobMetricsGet() {
    String jobName = "testJob";
    String jobId = "job_123";
    JobState jobState = new JobState(jobName, jobId);
    JobMetrics jobMetrics = JobMetrics.get(jobState);
    Assert.assertNotNull(jobMetrics.getMetricContext());
    List<Tag<?>> tags = jobMetrics.getMetricContext().getTags();
    Map<String, ?> tagMap = jobMetrics.getMetricContext().getTagMap();
    String contextId = tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME).toString();
    String contextName = tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME).toString();
    Assert.assertEquals(tagMap.size(), 4);
    Assert.assertEquals(tagMap.get(JobEvent.METADATA_JOB_ID), jobId);
    Assert.assertEquals(tagMap.get(JobEvent.METADATA_JOB_NAME), jobName);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME), contextId);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME), contextName);
    // should get the original jobMetrics, can check by the id
    JobMetrics jobMetrics1 = JobMetrics.get(jobName + "_", jobId);
    Assert.assertNotNull(jobMetrics1.getMetricContext());
    tagMap = jobMetrics1.getMetricContext().getTagMap();
    Assert.assertEquals(tags.size(), 4);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME), contextId);
    Assert.assertEquals(tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME), contextName);
    // remove original jobMetrics, should create a new one
    GobblinMetricsRegistry.getInstance().remove(jobMetrics.getId());
    JobMetrics jobMetrics2 = JobMetrics.get(jobName + "_", jobId);
    Assert.assertNotNull(jobMetrics2.getMetricContext());
    tagMap = jobMetrics2.getMetricContext().getTagMap();
    Assert.assertEquals(tags.size(), 4);
    Assert.assertNotEquals(tagMap.get(MetricContext.METRIC_CONTEXT_ID_TAG_NAME), contextId);
    Assert.assertNotEquals(tagMap.get(MetricContext.METRIC_CONTEXT_NAME_TAG_NAME), contextName);
}

Example 9 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class ClusterEventMetadataGeneratorTest method testProcessedCount.

@Test
public void testProcessedCount() throws Exception {
    JobContext jobContext = Mockito.mock(JobContext.class);
    JobState jobState = new JobState("jobName", "1234");
    TaskState taskState1 = new TaskState();
    TaskState taskState2 = new TaskState();
    taskState1.setTaskId("1");
    taskState1.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "1");
    taskState2.setTaskId("2");
    taskState2.setProp(ConfigurationKeys.WRITER_RECORDS_WRITTEN, "22");
    jobState.addTaskState(taskState1);
    jobState.addTaskState(taskState2);
    Mockito.when(jobContext.getJobState()).thenReturn(jobState);
    ClusterEventMetadataGenerator metadataGenerator = new ClusterEventMetadataGenerator();
    Map<String, String> metadata;
    // processed count is not in job cancel event
    metadata = metadataGenerator.getMetadata(jobContext, EventName.JOB_CANCEL);
    Assert.assertEquals(metadata.get("processedCount"), null);
    // processed count is in job complete event
    metadata = metadataGenerator.getMetadata(jobContext, EventName.getEnumFromEventId("JobCompleteTimer"));
    Assert.assertEquals(metadata.get("processedCount"), "23");
}

Example 10 with JobState

use of org.apache.gobblin.runtime.JobState in project incubator-gobblin by apache.

the class JobStateToJsonConverter method writeJobStates.

/**
 * Write a list of {@link JobState}s to json document.
 *
 * @param jsonWriter {@link com.google.gson.stream.JsonWriter}
 * @param jobStates list of {@link JobState}s to write to json document
 * @throws IOException
 */
private void writeJobStates(JsonWriter jsonWriter, List<? extends JobState> jobStates) throws IOException {
    jsonWriter.beginArray();
    for (JobState jobState : jobStates) {
        writeJobState(jsonWriter, jobState);
    }
    jsonWriter.endArray();
}