Examples with TezVertexID org.apache.tez.dag.records.TezVertexID used on opensource projects

Example 56 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestTaskExecution2 method createTaskRunner.

private TezTaskRunner2 createTaskRunner(ApplicationId appId, TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical, TaskReporter taskReporter, ListeningExecutorService executor, String processorClass, byte[] processorConf, boolean testRunner, boolean updateSysCounters) throws IOException {
    TezConfiguration tezConf = new TezConfiguration(defaultConf);
    UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
    Path testDir = new Path(workDir, UUID.randomUUID().toString());
    String[] localDirs = new String[] { testDir.toString() };
    TezDAGID dagId = TezDAGID.getInstance(appId, 1);
    TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
    TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
    TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
    ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass).setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
    TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", -1, processorDescriptor, new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null, null);
    TezExecutors sharedExecutor = new TezSharedExecutor(tezConf);
    TezTaskRunner2 taskRunner;
    if (testRunner) {
        taskRunner = new TezTaskRunner2ForTest(tezConf, ugi, localDirs, taskSpec, 1, new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String>create(), taskReporter, executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), updateSysCounters, sharedExecutor);
    } else {
        taskRunner = new TezTaskRunner2(tezConf, ugi, localDirs, taskSpec, 1, new HashMap<String, ByteBuffer>(), new HashMap<String, String>(), HashMultimap.<String, String>create(), taskReporter, executor, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), updateSysCounters, new DefaultHadoopShim(), sharedExecutor);
    }
    return taskRunner;
}

Example 57 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TezChild method cleanupOnTaskChanged.

/**
 * Cleans entries from the object registry, and resets the startedInputsMap if required
 *
 * @param containerTask
 *          the new task specification. Must be a valid task
 */
private void cleanupOnTaskChanged(ContainerTask containerTask) {
    Preconditions.checkState(!containerTask.shouldDie());
    Preconditions.checkState(containerTask.getTaskSpec() != null);
    TezVertexID newVertexID = containerTask.getTaskSpec().getTaskAttemptID().getTaskID().getVertexID();
    if (lastVertexID != null) {
        if (!lastVertexID.equals(newVertexID)) {
            objectRegistry.clearCache(ObjectRegistryImpl.ObjectLifeCycle.VERTEX);
        }
        if (!lastVertexID.getDAGId().equals(newVertexID.getDAGId())) {
            objectRegistry.clearCache(ObjectRegistryImpl.ObjectLifeCycle.DAG);
            startedInputsMap = HashMultimap.create();
        }
    }
    lastVertexID = newVertexID;
}

Example 58 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestLogicalIOProcessorRuntimeTask method testAutoStart.

@Test(timeout = 5000)
public void testAutoStart() throws Exception {
    TezDAGID dagId = createTezDagId();
    TezVertexID vertexId = createTezVertexId(dagId);
    Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<String, ByteBuffer>();
    Multimap<String, String> startedInputsMap = HashMultimap.create();
    TezUmbilical umbilical = mock(TezUmbilical.class);
    TezConfiguration tezConf = new TezConfiguration();
    tezConf.set(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ALLOCATOR_CLASS, ScalingAllocator.class.getName());
    TezTaskAttemptID taId1 = createTaskAttemptID(vertexId, 1);
    TaskSpec task1 = createTaskSpec(taId1, "dag1", "vertex1", 30);
    TezTaskAttemptID taId2 = createTaskAttemptID(vertexId, 2);
    TaskSpec task2 = createTaskSpec(taId2, "dag2", "vertex1", 10);
    TezSharedExecutor sharedExecutor = new TezSharedExecutor(tezConf);
    LogicalIOProcessorRuntimeTask lio1 = new LogicalIOProcessorRuntimeTask(task1, 0, tezConf, null, umbilical, serviceConsumerMetadata, new HashMap<String, String>(), startedInputsMap, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), true, new DefaultHadoopShim(), sharedExecutor);
    try {
        lio1.initialize();
        lio1.run();
        lio1.close();
        // Input should've been started, Output should not have been started
        assertEquals(1, TestProcessor.runCount);
        assertEquals(1, TestInput.startCount);
        assertEquals(0, TestOutput.startCount);
        // test that invocations of progress are counted correctly
        assertEquals(true, lio1.getAndClearProgressNotification());
        // cleared after getting
        assertEquals(false, lio1.getAndClearProgressNotification());
        assertEquals(30, TestInput.vertexParallelism);
        assertEquals(0, TestOutput.vertexParallelism);
        assertEquals(30, lio1.getProcessorContext().getVertexParallelism());
        assertEquals(30, lio1.getInputContexts().iterator().next().getVertexParallelism());
        assertEquals(30, lio1.getOutputContexts().iterator().next().getVertexParallelism());
    } catch (Exception e) {
        fail();
        sharedExecutor.shutdownNow();
    } finally {
        cleanupAndTest(lio1);
    }
    // local mode
    tezConf.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
    LogicalIOProcessorRuntimeTask lio2 = new LogicalIOProcessorRuntimeTask(task2, 0, tezConf, null, umbilical, serviceConsumerMetadata, new HashMap<String, String>(), startedInputsMap, null, "", new ExecutionContextImpl("localhost"), Runtime.getRuntime().maxMemory(), true, new DefaultHadoopShim(), sharedExecutor);
    try {
        lio2.initialize();
        lio2.run();
        lio2.close();
        // Input should not have been started again, Output should not have been started
        assertEquals(2, TestProcessor.runCount);
        assertEquals(1, TestInput.startCount);
        assertEquals(0, TestOutput.startCount);
        assertEquals(30, TestInput.vertexParallelism);
        assertEquals(0, TestOutput.vertexParallelism);
        // Check if parallelism is available in processor/ i/p / o/p contexts
        assertEquals(10, lio2.getProcessorContext().getVertexParallelism());
        assertEquals(10, lio2.getInputContexts().iterator().next().getVertexParallelism());
        assertEquals(10, lio2.getOutputContexts().iterator().next().getVertexParallelism());
    } catch (Exception e) {
        fail();
    } finally {
        cleanupAndTest(lio2);
        sharedExecutor.shutdownNow();
    }
}

Example 59 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestMockDAGAppMaster method testInternalPreemption.

@Test(timeout = 5000)
public void testInternalPreemption() throws Exception {
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
    tezClient.start();
    MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
    MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
    mockLauncher.startScheduling(false);
    // there is only 1 task whose first attempt will be preempted
    DAG dag = DAG.create("testInternalPreemption");
    Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 1);
    dag.addVertex(vA);
    DAGClient dagClient = tezClient.submitDAG(dag);
    mockLauncher.waitTillContainersLaunched();
    ContainerData cData = mockLauncher.getContainers().values().iterator().next();
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    mockApp.getTaskSchedulerManager().preemptContainer(0, cData.cId);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), 0);
    TezTaskAttemptID killedTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    TaskAttempt killedTa = dagImpl.getVertex(vA.getName()).getTask(0).getAttempt(killedTaId);
    Assert.assertEquals(TaskAttemptState.KILLED, killedTa.getState());
    tezClient.stop();
}

Example 60 with TezVertexID

use of org.apache.tez.dag.records.TezVertexID in project tez by apache.

the class TestPreemption method testPreemptionJob.

void testPreemptionJob(MockTezClient tezClient, DAG dag, int vertexIndex, int upToTaskVersion, String info) throws Exception {
    System.out.println("TestPreemption - Running - " + info);
    TezConfiguration tezconf = new TezConfiguration(defaultConf);
    tezconf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 0);
    // turn off scheduling to block DAG before submitting it
    mockLauncher.startScheduling(false);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGImpl dagImpl = (DAGImpl) mockApp.getContext().getCurrentDAG();
    TezVertexID vertexId = TezVertexID.getInstance(dagImpl.getID(), vertexIndex);
    TezTaskAttemptID taId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), 0);
    mockLauncher.preemptContainerForTask(taId.getTaskID(), upToTaskVersion);
    mockLauncher.startScheduling(true);
    dagClient.waitForCompletion();
    Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
    for (int i = 0; i <= upToTaskVersion; ++i) {
        TezTaskAttemptID testTaId = TezTaskAttemptID.getInstance(TezTaskID.getInstance(vertexId, 0), i);
        TaskAttemptImpl taImpl = dagImpl.getTaskAttempt(testTaId);
        Assert.assertEquals(TaskAttemptStateInternal.KILLED, taImpl.getInternalState());
        Assert.assertEquals(TaskAttemptTerminationCause.EXTERNAL_PREEMPTION, taImpl.getTerminationCause());
    }
    System.out.println("TestPreemption - Done running - " + info);
}