Examples with TezSharedExecutor org.apache.tez.common.TezSharedExecutor used on opensource projects

Example 6 with TezSharedExecutor

use of org.apache.tez.common.TezSharedExecutor in project tez by apache.

the class TestTaskExecution2 method createTaskRunner.

private TezTaskRunner2 createTaskRunner(ApplicationId appId, TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical, TaskReporter taskReporter, ListeningExecutorService executor, String processorClass, byte[] processorConf, boolean testRunner, boolean updateSysCounters) throws IOException {
    TezConfiguration tezConf = new TezConfiguration(defaultConf);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    Path testDir = new Path(workDir, UUID.randomUUID().toString());
    String[] localDirs = new String[] { testDir.toString() };
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass).setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor, new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null, null);
    TezExecutors sharedExecutor = new TezSharedExecutor(tezConf);
    TezTaskRunner2 taskRunner;
    if (testRunner) {
        taskRunner = new TezTaskRunner2ForTest(tezConf, ugi, localDirs, taskSpec, 1, new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String>create(), taskReporter, executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), updateSysCounters, sharedExecutor);
    } else {
        taskRunner = new TezTaskRunner2(tezConf, ugi, localDirs, taskSpec, 1, new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String>create(), taskReporter, executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), updateSysCounters, new DefaultHadoopShim(), sharedExecutor);
    }
    return taskRunner;
}

Example 7 with TezSharedExecutor

use of org.apache.tez.common.TezSharedExecutor in project tez by apache.

the class TestLogicalIOProcessorRuntimeTask method testAutoStart.

@Test(timeout = 5000)
public void testAutoStart() throws Exception {
    TezDAGID dagId = createTezDagId();
    TezVertexID vertexId = createTezVertexId(dagId);
    Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS, ScalingAllocator.class.getName());
    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "dag1", "vertex1", 30);
    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "dag2", "vertex1", 10);
    TezSharedExecutor sharedExecutor = new TezSharedExecutor(tezConf);
    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf, null, umbilical, serviceConsumerMetadata, new HashMap<String, String>(), startedInputsMap, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), true, new DefaultHadoopShim(), sharedExecutor);
    try {
        lio1.initialize();
        lio1.run();
        lio1.close();
        // Input should've been started, Output should not have been started
        assertEquals(1, TestProcessor.runCount);
        assertEquals(1, TestInput.startCount);
        assertEquals(0, TestOutput.startCount);
        // test that invocations of progress are counted correctly
        assertEquals(true, lio1.getAndClearProgressNotification());
        // cleared after getting
        assertEquals(false, lio1.getAndClearProgressNotification());
        assertEquals(30, TestInput.vertexParallelism);
        assertEquals(0, TestOutput.vertexParallelism);
        assertEquals(30, lio1.getProcessorContext().getVertexParallelism());
        assertEquals(30, lio1.getInputContexts().iterator().next().getVertexParallelism());
        assertEquals(30, lio1.getOutputContexts().iterator().next().getVertexParallelism());
    } catch (Exception e) {
        fail();
        sharedExecutor.shutdownNow();
    } finally {
        cleanupAndTest(lio1);
    }
    // local mode
    tezConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    LogicalIOProcessorRuntimeTask lio2 = new LogicalIOProcessorRuntimeTask(task2, 0, tezConf, null, umbilical, serviceConsumerMetadata, new HashMap<String, String>(), startedInputsMap, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), true, new DefaultHadoopShim(), sharedExecutor);
    try {
        lio2.initialize();
        lio2.run();
        lio2.close();
        // Input should not have been started again, Output should not have been started
        assertEquals(2, TestProcessor.runCount);
        assertEquals(1, TestInput.startCount);
        assertEquals(0, TestOutput.startCount);
        assertEquals(30, TestInput.vertexParallelism);
        assertEquals(0, TestOutput.vertexParallelism);
        // Check if parallelism is available in processor/ i/p / o/p contexts
        assertEquals(10, lio2.getProcessorContext().getVertexParallelism());
        assertEquals(10, lio2.getInputContexts().iterator().next().getVertexParallelism());
        assertEquals(10, lio2.getOutputContexts().iterator().next().getVertexParallelism());
    } catch (Exception e) {
        fail();
    } finally {
        cleanupAndTest(lio2);
        sharedExecutor.shutdownNow();
    }
}

Example 8 with TezSharedExecutor

use of org.apache.tez.common.TezSharedExecutor in project tez by apache.

the class TestTezTaskRunner2 method testTaskConfUsage.

@Test(timeout = 5000)
public void testTaskConfUsage() throws Exception {
    Configuration conf = new Configuration(false);
    conf.set("global", "global1");
    conf.set("global_override", "global1");
    String[] localDirs = null;
    Configuration taskConf = new Configuration(false);
    conf.set("global_override", "task1");
    conf.set("task", "task1");
    List<InputSpec> inputSpecList = new ArrayList<>();
    List<OutputSpec> outputSpecList = new ArrayList<>();
    TaskSpec taskSpec = new TaskSpec("dagName", "vertexName", 1, mock(ProcessorDescriptor.class), inputSpecList, outputSpecList, null, taskConf);
    TezExecutors sharedExecutor = new TezSharedExecutor(conf);
    TezTaskRunner2 taskRunner2 = new TezTaskRunner2(conf, mock(UserGroupInformation.class), localDirs, taskSpec, 1, null, null, null, mock(TaskReporter.class), null, null, "pid", null, 1000, false, new DefaultHadoopShim(), sharedExecutor);
    Assert.assertEquals("global1", taskRunner2.task.getTaskConf().get("global"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("global_override"));
    Assert.assertEquals("task1", taskRunner2.task.getTaskConf().get("task"));
    sharedExecutor.shutdownNow();
}

Example 9 with TezSharedExecutor

use of org.apache.tez.common.TezSharedExecutor in project tez by apache.

the class TestProcessorContext method testDagNumber.

@Test(timeout = 5000)
public void testDagNumber() throws IOException {
    String[] localDirs = new String[] { "dummyLocalDir" };
    int appAttemptNumber = 1;
    TezUmbilical tezUmbilical = mock(TezUmbilical.class);
    String dagName = "DAG_NAME";
    String vertexName = "VERTEX_NAME";
    int vertexParallelism = 20;
    int dagNumber = 52;
    ApplicationId appId = ApplicationId.newInstance(10000, 13);
    TezDAGID dagId = TezDAGID.getInstance(appId, dagNumber);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 6);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 4);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 2);
    TaskSpec mockSpec = mock(TaskSpec.class);
    when(mockSpec.getInputs()).thenReturn(Collections.singletonList(mock(InputSpec.class)));
    when(mockSpec.getOutputs()).thenReturn(Collections.singletonList(mock(OutputSpec.class)));
    Configuration conf = new Configuration();
    TezSharedExecutor sharedExecutor = new TezSharedExecutor(conf);
    LogicalIOProcessorRuntimeTask runtimeTask = new LogicalIOProcessorRuntimeTask(mockSpec, 1, conf, new String[] { "/" }, tezUmbilical, null, null, null, null, "", null, 1024, false, new DefaultHadoopShim(), sharedExecutor);
    LogicalIOProcessorRuntimeTask mockTask = spy(runtimeTask);
    Map<String, ByteBuffer> serviceConsumerMetadata = Maps.newHashMap();
    Map<String, String> auxServiceEnv = Maps.newHashMap();
    MemoryDistributor memDist = mock(MemoryDistributor.class);
    ProcessorDescriptor processorDesc = mock(ProcessorDescriptor.class);
    InputReadyTracker inputReadyTracker = mock(InputReadyTracker.class);
    ObjectRegistry objectRegistry = new ObjectRegistryImpl();
    ExecutionContext execContext = new ExecutionContextImpl("localhost");
    long memAvailable = 10000l;
    TezProcessorContextImpl procContext = new TezProcessorContextImpl(new Configuration(), localDirs, appAttemptNumber, tezUmbilical, dagName, vertexName, vertexParallelism, taskAttemptId, null, runtimeTask, serviceConsumerMetadata, auxServiceEnv, memDist, processorDesc, inputReadyTracker, objectRegistry, execContext, memAvailable, sharedExecutor);
    assertEquals(dagNumber, procContext.getDagIdentifier());
    assertEquals(appAttemptNumber, procContext.getDAGAttemptNumber());
    assertEquals(appId, procContext.getApplicationId());
    assertEquals(dagName, procContext.getDAGName());
    assertEquals(vertexName, procContext.getTaskVertexName());
    assertEquals(vertexId.getId(), procContext.getTaskVertexIndex());
    assertTrue(Arrays.equals(localDirs, procContext.getWorkDirs()));
    // test auto call of notifyProgress
    procContext.setProgress(0.1f);
    verify(mockTask, times(1)).notifyProgressInvocation();
    sharedExecutor.shutdown();
}

Example 10 with TezSharedExecutor

use of org.apache.tez.common.TezSharedExecutor in project tez by apache.

the class TestShuffleInputEventHandlerOrderedGrouped method setup.

@Before
public void setup() throws Exception {
    sharedExecutor = new TezSharedExecutor(new Configuration());
    setupScheduler(2);
}