Examples with MRTaskInfo io.cdap.cdap.proto.MRTaskInfo used on opensource projects

Example 1 with MRTaskInfo

use of io.cdap.cdap.proto.MRTaskInfo in project cdap by caskdata.

the class LocalMRJobInfoFetcher method getMRJobInfo.

/**
 * @param runId for which information will be returned.
 * @return a {@link MRJobInfo} containing information about a particular MapReduce program run.
 */
@Override
public MRJobInfo getMRJobInfo(Id.Run runId) throws IOException {
    Preconditions.checkArgument(ProgramType.MAPREDUCE.equals(runId.getProgram().getType()));
    // baseTags has tag keys: ns.app.mr.runid
    Map<String, String> baseTags = Maps.newHashMap();
    baseTags.put(Constants.Metrics.Tag.NAMESPACE, runId.getNamespace().getId());
    baseTags.put(Constants.Metrics.Tag.APP, runId.getProgram().getApplicationId());
    baseTags.put(Constants.Metrics.Tag.MAPREDUCE, runId.getProgram().getId());
    baseTags.put(Constants.Metrics.Tag.RUN_ID, runId.getId());
    Map<String, String> mapTags = Maps.newHashMap(baseTags);
    mapTags.put(Constants.Metrics.Tag.MR_TASK_TYPE, MapReduceMetrics.TaskType.Mapper.getId());
    Map<String, String> reduceTags = Maps.newHashMap(baseTags);
    reduceTags.put(Constants.Metrics.Tag.MR_TASK_TYPE, MapReduceMetrics.TaskType.Reducer.getId());
    // map from RunId -> (CounterName -> CounterValue)
    Table<String, String, Long> mapTaskMetrics = HashBasedTable.create();
    Table<String, String, Long> reduceTaskMetrics = HashBasedTable.create();
    // Populate mapTaskMetrics and reduce Task Metrics via MetricStore. Used to construct MRTaskInfo below.
    Map<String, String> metricNamesToCounters = Maps.newHashMap();
    metricNamesToCounters.put(prependSystem(MapReduceMetrics.METRIC_TASK_INPUT_RECORDS), TaskCounter.MAP_INPUT_RECORDS.name());
    metricNamesToCounters.put(prependSystem(MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS), TaskCounter.MAP_OUTPUT_RECORDS.name());
    metricNamesToCounters.put(prependSystem(MapReduceMetrics.METRIC_TASK_BYTES), TaskCounter.MAP_OUTPUT_BYTES.name());
    metricNamesToCounters.put(prependSystem(MapReduceMetrics.METRIC_TASK_COMPLETION), MapReduceMetrics.METRIC_TASK_COMPLETION);
    // get metrics grouped by instance-id for the map tasks
    queryGroupedAggregates(mapTags, mapTaskMetrics, metricNamesToCounters);
    Map<String, Long> mapProgress = Maps.newHashMap();
    if (mapTaskMetrics.columnMap().containsKey(MapReduceMetrics.METRIC_TASK_COMPLETION)) {
        mapProgress = Maps.newHashMap(mapTaskMetrics.columnMap().remove(MapReduceMetrics.METRIC_TASK_COMPLETION));
    }
    Map<String, String> reduceMetricsToCounters = Maps.newHashMap();
    reduceMetricsToCounters.put(prependSystem(MapReduceMetrics.METRIC_TASK_INPUT_RECORDS), TaskCounter.REDUCE_INPUT_RECORDS.name());
    reduceMetricsToCounters.put(prependSystem(MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS), TaskCounter.REDUCE_OUTPUT_RECORDS.name());
    reduceMetricsToCounters.put(prependSystem(MapReduceMetrics.METRIC_TASK_COMPLETION), MapReduceMetrics.METRIC_TASK_COMPLETION);
    // get metrics grouped by instance-id for the map tasks
    queryGroupedAggregates(reduceTags, reduceTaskMetrics, reduceMetricsToCounters);
    Map<String, Long> reduceProgress = Maps.newHashMap();
    if (reduceTaskMetrics.columnMap().containsKey(MapReduceMetrics.METRIC_TASK_COMPLETION)) {
        reduceProgress = Maps.newHashMap(reduceTaskMetrics.columnMap().remove(MapReduceMetrics.METRIC_TASK_COMPLETION));
    }
    // Construct MRTaskInfos from the information we can get from Metric system.
    List<MRTaskInfo> mapTaskInfos = Lists.newArrayList();
    for (Map.Entry<String, Map<String, Long>> taskEntry : mapTaskMetrics.rowMap().entrySet()) {
        String mapTaskId = taskEntry.getKey();
        mapTaskInfos.add(new MRTaskInfo(mapTaskId, null, null, null, mapProgress.get(mapTaskId) / 100.0F, taskEntry.getValue()));
    }
    List<MRTaskInfo> reduceTaskInfos = Lists.newArrayList();
    for (Map.Entry<String, Map<String, Long>> taskEntry : reduceTaskMetrics.rowMap().entrySet()) {
        String reduceTaskId = taskEntry.getKey();
        reduceTaskInfos.add(new MRTaskInfo(reduceTaskId, null, null, null, reduceProgress.get(reduceTaskId) / 100.0F, taskEntry.getValue()));
    }
    return getJobCounters(mapTags, reduceTags, mapTaskInfos, reduceTaskInfos, runId);
}

Example 2 with MRTaskInfo

use of io.cdap.cdap.proto.MRTaskInfo in project cdap by caskdata.

the class LocalMRJobInfoFetcherTest method testGetMRJobInfo.

@Test
public void testGetMRJobInfo() throws Exception {
    Id.Program programId = Id.Program.from("fooNamespace", "testApp", ProgramType.MAPREDUCE, "fooMapReduce");
    Id.Run runId = new Id.Run(programId, "run10878");
    Map<String, String> runContext = ImmutableMap.of(Constants.Metrics.Tag.NAMESPACE, programId.getNamespaceId(), Constants.Metrics.Tag.APP, programId.getApplicationId(), Constants.Metrics.Tag.MAPREDUCE, programId.getId(), Constants.Metrics.Tag.RUN_ID, runId.getId());
    Map<String, String> mapTypeContext = addToContext(runContext, Constants.Metrics.Tag.MR_TASK_TYPE, MapReduceMetrics.TaskType.Mapper.getId());
    Map<String, String> reduceTypeContext = addToContext(runContext, Constants.Metrics.Tag.MR_TASK_TYPE, MapReduceMetrics.TaskType.Reducer.getId());
    String mapTask1Name = "task_m_01";
    Map<String, String> mapTask1Context = addToContext(mapTypeContext, Constants.Metrics.Tag.INSTANCE_ID, mapTask1Name);
    String mapTask2Name = "task_m_02";
    Map<String, String> mapTask2Context = addToContext(mapTypeContext, Constants.Metrics.Tag.INSTANCE_ID, mapTask2Name);
    String reduceTaskName = "task_r_01";
    Map<String, String> reduceTaskContext = addToContext(reduceTypeContext, Constants.Metrics.Tag.INSTANCE_ID, reduceTaskName);
    // Imitate a MapReduce Job running (gauge mapper and reducer metrics)
    long measureTime = System.currentTimeMillis() / 1000;
    gauge(mapTypeContext, MapReduceMetrics.METRIC_COMPLETION, measureTime, 76L);
    gauge(reduceTypeContext, MapReduceMetrics.METRIC_COMPLETION, measureTime, 52L);
    gauge(mapTask1Context, MapReduceMetrics.METRIC_TASK_COMPLETION, measureTime, 100L);
    gauge(mapTask1Context, MapReduceMetrics.METRIC_TASK_INPUT_RECORDS, measureTime, 32L);
    gauge(mapTask1Context, MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS, measureTime, 320L);
    gauge(mapTask2Context, MapReduceMetrics.METRIC_TASK_COMPLETION, measureTime, 12L);
    gauge(mapTask2Context, MapReduceMetrics.METRIC_TASK_INPUT_RECORDS, measureTime, 6L);
    gauge(mapTask2Context, MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS, measureTime, 60L);
    // gauge job-level counters for mappers
    gauge(mapTypeContext, MapReduceMetrics.METRIC_INPUT_RECORDS, measureTime, 38L);
    gauge(mapTypeContext, MapReduceMetrics.METRIC_OUTPUT_RECORDS, measureTime, 380L);
    gauge(reduceTaskContext, MapReduceMetrics.METRIC_TASK_COMPLETION, measureTime, 76L);
    gauge(reduceTaskContext, MapReduceMetrics.METRIC_TASK_INPUT_RECORDS, measureTime, 320L);
    gauge(reduceTaskContext, MapReduceMetrics.METRIC_TASK_OUTPUT_RECORDS, measureTime, 1L);
    // gauge job-level counters for reducers
    gauge(reduceTypeContext, MapReduceMetrics.METRIC_INPUT_RECORDS, measureTime, 320L);
    gauge(reduceTypeContext, MapReduceMetrics.METRIC_OUTPUT_RECORDS, measureTime, 1L);
    LocalMRJobInfoFetcher localMRJobInfoFetcher = injector.getInstance(LocalMRJobInfoFetcher.class);
    MRJobInfo mrJobInfo = localMRJobInfoFetcher.getMRJobInfo(runId);
    // Incomplete because MapReduceMetricsInfo does not provide task-level state and start/end times.
    Assert.assertFalse(mrJobInfo.isComplete());
    // Check job-level counters
    Map<String, Long> jobCounters = mrJobInfo.getCounters();
    Assert.assertEquals((Long) 38L, jobCounters.get(TaskCounter.MAP_INPUT_RECORDS.name()));
    Assert.assertEquals((Long) 380L, jobCounters.get(TaskCounter.MAP_OUTPUT_RECORDS.name()));
    Assert.assertEquals((Long) 320L, jobCounters.get(TaskCounter.REDUCE_INPUT_RECORDS.name()));
    Assert.assertEquals((Long) 1L, jobCounters.get(TaskCounter.REDUCE_OUTPUT_RECORDS.name()));
    // Ensure all tasks show up
    List<MRTaskInfo> mapTasks = mrJobInfo.getMapTasks();
    List<MRTaskInfo> reduceTasks = mrJobInfo.getReduceTasks();
    Assert.assertEquals(2, mapTasks.size());
    Assert.assertEquals(1, reduceTasks.size());
    MRTaskInfo mapTask1 = findByTaskId(mapTasks, mapTask1Name);
    MRTaskInfo mapTask2 = findByTaskId(mapTasks, mapTask2Name);
    MRTaskInfo reduceTask = findByTaskId(reduceTasks, reduceTaskName);
    // Check task-level counters
    Map<String, Long> mapTask1Counters = mapTask1.getCounters();
    Assert.assertEquals((Long) 32L, mapTask1Counters.get(TaskCounter.MAP_INPUT_RECORDS.name()));
    Assert.assertEquals((Long) 320L, mapTask1Counters.get(TaskCounter.MAP_OUTPUT_RECORDS.name()));
    Map<String, Long> mapTask2Counters = mapTask2.getCounters();
    Assert.assertEquals((Long) 6L, mapTask2Counters.get(TaskCounter.MAP_INPUT_RECORDS.name()));
    Assert.assertEquals((Long) 60L, mapTask2Counters.get(TaskCounter.MAP_OUTPUT_RECORDS.name()));
    Map<String, Long> reduceTaskCounters = reduceTask.getCounters();
    Assert.assertEquals((Long) 320L, reduceTaskCounters.get(TaskCounter.REDUCE_INPUT_RECORDS.name()));
    Assert.assertEquals((Long) 1L, reduceTaskCounters.get(TaskCounter.REDUCE_OUTPUT_RECORDS.name()));
    // Checking progress
    float permittedProgressDelta = 0.01F;
    Assert.assertEquals(0.76F, mrJobInfo.getMapProgress(), permittedProgressDelta);
    Assert.assertEquals(0.52F, mrJobInfo.getReduceProgress(), permittedProgressDelta);
    Assert.assertEquals(1.0F, mapTask1.getProgress(), permittedProgressDelta);
    Assert.assertEquals(0.12F, mapTask2.getProgress(), permittedProgressDelta);
    Assert.assertEquals(0.76F, reduceTask.getProgress(), permittedProgressDelta);
}