Examples with TezExecutors org.apache.tez.common.TezExecutors used on opensource projects

Example 1 with TezExecutors

use of org.apache.tez.common.TezExecutors in project tez by apache.

the class TestTaskExecution2 method createTaskRunner.

private TezTaskRunner2 createTaskRunner(ApplicationId appId, TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical, TaskReporter taskReporter, ListeningExecutorService executor, String processorClass, byte[] processorConf, boolean testRunner, boolean updateSysCounters) throws IOException {
    TezConfiguration tezConf = new TezConfiguration(defaultConf);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    Path testDir = new Path(workDir, UUID.randomUUID().toString());
    String[] localDirs = new String[] { testDir.toString() };
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass).setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor, new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null, null);
    TezExecutors sharedExecutor = new TezSharedExecutor(tezConf);
    TezTaskRunner2 taskRunner;
    if (testRunner) {
        taskRunner = new TezTaskRunner2ForTest(tezConf, ugi, localDirs, taskSpec, 1, new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String>create(), taskReporter, executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), updateSysCounters, sharedExecutor);
    } else {
        taskRunner = new TezTaskRunner2(tezConf, ugi, localDirs, taskSpec, 1, new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String>create(), taskReporter, executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), updateSysCounters, new DefaultHadoopShim(), sharedExecutor);
    }
    return taskRunner;
}

Example 2 with TezExecutors

use of org.apache.tez.common.TezExecutors in project tez by apache.

the class TestTezTaskRunner2 method testTaskConfUsage.

@Test(timeout = 5000)
public void testTaskConfUsage() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set("global", "global1");
    conf.set("global_override", "global1");
    String[] localDirs = null;
    Configuration taskConf = new Configuration(false);
    conf.set("global_override", "task1");
    conf.set("task", "task1");
    List<InputSpec> inputSpecList = new ArrayList<>();
    List<OutputSpec> outputSpecList = new ArrayList<>();
    TaskSpec taskSpec = new TaskSpec("dagName", "vertexName", 1, mock(ProcessorDescriptor.class), inputSpecList, outputSpecList, null, taskConf);
    TezExecutors sharedExecutor = new TezSharedExecutor(conf);
    TezTaskRunner2 taskRunner2 = new TezTaskRunner2(conf, mock(UserGroupInformation.class), localDirs, taskSpec, 1, null, null, null, mock(TaskReporter.class), null, null, "pid", null, 1000, false, new DefaultHadoopShim(), sharedExecutor);
    Assert.assertEquals("global1", taskRunner2.task.getTaskConf().get("global"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("global_override"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("task"));
    sharedExecutor.shutdownNow();
}