Examples with TaskContext org.apache.tez.runtime.api.TaskContext used on opensource projects

Example 1 with TaskContext

use of org.apache.tez.runtime.api.TaskContext in project tez by apache.

the class TestMRCombiner method getTaskContext.

private TaskContext getTaskContext(TezConfiguration conf) throws IOException {
    UserPayload payload = TezUtils.createUserPayloadFromConf(conf);
    TaskContext taskContext = Mockito.mock(InputContext.class);
    Mockito.when(taskContext.getUserPayload()).thenReturn(payload);
    Mockito.when(taskContext.getCounters()).thenReturn(new TezCounters());
    Mockito.when(taskContext.getApplicationId()).thenReturn(ApplicationId.newInstance(123456, 1));
    return taskContext;
}

Example 2 with TaskContext

use of org.apache.tez.runtime.api.TaskContext in project tez by apache.

the class TestMRCombiner method testRunNewCombiner.

@Test
public void testRunNewCombiner() throws IOException, InterruptedException {
    TezConfiguration conf = new TezConfiguration();
    setKeyAndValueClassTypes(conf);
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setClass(MRJobConfig.COMBINE_CLASS_ATTR, NewReducer.class, Object.class);
    TaskContext taskContext = getTaskContext(conf);
    MRCombiner combiner = new MRCombiner(taskContext);
    Writer writer = Mockito.mock(Writer.class);
    combiner.combine(new TezRawKeyValueIteratorTest(), writer);
    long inputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS).getValue();
    long outputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS).getValue();
    assertEquals(6, inputRecords);
    assertEquals(3, outputRecords);
    // verify combiner output keys and values
    verifyKeyAndValues(writer);
}

Example 3 with TaskContext

use of org.apache.tez.runtime.api.TaskContext in project tez by apache.

the class TestMRCombiner method testRunOldCombiner.

@Test
public void testRunOldCombiner() throws IOException, InterruptedException {
    TezConfiguration conf = new TezConfiguration();
    setKeyAndValueClassTypes(conf);
    conf.setClass("mapred.combiner.class", OldReducer.class, Object.class);
    TaskContext taskContext = getTaskContext(conf);
    MRCombiner combiner = new MRCombiner(taskContext);
    Writer writer = Mockito.mock(Writer.class);
    combiner.combine(new TezRawKeyValueIteratorTest(), writer);
    long inputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS).getValue();
    long outputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS).getValue();
    assertEquals(6, inputRecords);
    assertEquals(3, outputRecords);
    // verify combiner output keys and values
    verifyKeyAndValues(writer);
}

Example 4 with TaskContext

use of org.apache.tez.runtime.api.TaskContext in project tez by apache.

the class TezRuntimeUtils method instantiateCombiner.

@SuppressWarnings("unchecked")
public static Combiner instantiateCombiner(Configuration conf, TaskContext taskContext) throws IOException {
    Class<? extends Combiner> clazz;
    String className = conf.get(TezRuntimeConfiguration.TEZ_RUNTIME_COMBINER_CLASS);
    if (className == null) {
        return null;
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Using Combiner class: " + className);
    }
    try {
        clazz = (Class<? extends Combiner>) conf.getClassByName(className);
    } catch (ClassNotFoundException e) {
        throw new IOException("Unable to load combiner class: " + className);
    }
    Combiner combiner = null;
    Constructor<? extends Combiner> ctor;
    try {
        ctor = clazz.getConstructor(TaskContext.class);
        combiner = ctor.newInstance(taskContext);
    } catch (SecurityException e) {
        throw new IOException(e);
    } catch (NoSuchMethodException e) {
        throw new IOException(e);
    } catch (IllegalArgumentException e) {
        throw new IOException(e);
    } catch (InstantiationException e) {
        throw new IOException(e);
    } catch (IllegalAccessException e) {
        throw new IOException(e);
    } catch (InvocationTargetException e) {
        throw new IOException(e);
    }
    return combiner;
}

Example 5 with TaskContext

use of org.apache.tez.runtime.api.TaskContext in project tez by apache.

the class TestMRCombiner method testTop2RunNewCombiner.

@Test
public void testTop2RunNewCombiner() throws IOException, InterruptedException {
    TezConfiguration conf = new TezConfiguration();
    setKeyAndValueClassTypes(conf);
    conf.setBoolean("mapred.mapper.new-api", true);
    conf.setClass(MRJobConfig.COMBINE_CLASS_ATTR, Top2NewReducer.class, Object.class);
    TaskContext taskContext = getTaskContext(conf);
    MRCombiner combiner = new MRCombiner(taskContext);
    Writer writer = Mockito.mock(Writer.class);
    combiner.combine(new TezRawKeyValueIteratorTest(), writer);
    long inputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_INPUT_RECORDS).getValue();
    long outputRecords = taskContext.getCounters().findCounter(TaskCounter.COMBINE_OUTPUT_RECORDS).getValue();
    assertEquals(6, inputRecords);
    assertEquals(5, outputRecords);
}