Examples with Vertex org.apache.tez.dag.app.dag.Vertex used on opensource projects

Example 21 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestDAGScheduler method testConcurrencyLimit.

@Test(timeout = 5000)
public void testConcurrencyLimit() {
    MockEventHandler mockEventHandler = new MockEventHandler();
    DAG mockDag = mock(DAG.class);
    when(mockDag.getTotalVertices()).thenReturn(2);
    TezVertexID vId0 = TezVertexID.fromString("vertex_1436907267600_195589_1_00");
    TezVertexID vId1 = TezVertexID.fromString("vertex_1436907267600_195589_1_01");
    TezTaskID tId0 = TezTaskID.getInstance(vId0, 0);
    TezTaskID tId1 = TezTaskID.getInstance(vId1, 0);
    TaskAttempt mockAttempt;
    Vertex mockVertex = mock(Vertex.class);
    when(mockDag.getVertex((TezVertexID) any())).thenReturn(mockVertex);
    when(mockVertex.getDistanceFromRoot()).thenReturn(0);
    when(mockVertex.getVertexId()).thenReturn(vId0);
    DAGScheduler scheduler = new DAGSchedulerNaturalOrder(mockDag, mockEventHandler);
    // not effective
    scheduler.addVertexConcurrencyLimit(vId0, 0);
    // schedule beyond limit and it gets scheduled
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 0));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(1, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 1));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(2, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId0, 2));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    Assert.assertEquals(3, mockEventHandler.events.size());
    mockEventHandler.events.clear();
    List<TaskAttempt> mockAttempts = Lists.newArrayList();
    int completed = 0;
    int requested = 0;
    int scheduled = 0;
    // effective
    scheduler.addVertexConcurrencyLimit(vId1, 2);
    // schedule beyond limit and it gets buffered
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // buffered
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
    scheduler.taskCompleted(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_COMPLETED, mockAttempts.get(completed++)));
    // no extra scheduling
    Assert.assertEquals(scheduled, mockEventHandler.events.size());
    mockAttempt = mock(TaskAttempt.class);
    mockAttempts.add(mockAttempt);
    when(mockAttempt.getID()).thenReturn(TezTaskAttemptID.getInstance(tId1, requested++));
    scheduler.scheduleTask(new DAGEventSchedulerUpdate(DAGEventSchedulerUpdate.UpdateType.TA_SCHEDULE, mockAttempt));
    // scheduled
    Assert.assertEquals(scheduled + 1, mockEventHandler.events.size());
    Assert.assertEquals(mockAttempts.get(scheduled).getID(), // matches order
    mockEventHandler.events.get(scheduled).getTaskAttemptID());
    scheduled++;
}

Example 22 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestEdge method testEdgeManagerPluginCtxGetVertexGroupName.

@Test(timeout = 5000)
public void testEdgeManagerPluginCtxGetVertexGroupName() throws TezException {
    EdgeManagerPluginDescriptor edgeManagerDescriptor = EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
    EdgeProperty edgeProp = EdgeProperty.create(edgeManagerDescriptor, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create("Out"), InputDescriptor.create("In"));
    Edge edge = new Edge(edgeProp, null, null);
    Vertex srcV = mock(Vertex.class), destV = mock(Vertex.class);
    String srcName = "srcV", destName = "destV";
    when(srcV.getName()).thenReturn(srcName);
    when(destV.getName()).thenReturn(destName);
    edge.setSourceVertex(srcV);
    edge.setDestinationVertex(destV);
    assertNull(edge.edgeManager.getContext().getVertexGroupName());
    String group = "group";
    when(destV.getGroupInputSpecList()).thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList("v1", "v3"), null)));
    assertNull(edge.edgeManager.getContext().getVertexGroupName());
    when(destV.getGroupInputSpecList()).thenReturn(Arrays.asList(new GroupInputSpec(group, Arrays.asList(srcName, "v3"), null)));
    assertEquals(group, edge.edgeManager.getContext().getVertexGroupName());
}

Example 23 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestTaskImpl method setup.

@Before
public void setup() {
    conf = new Configuration();
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 4);
    taskCommunicatorManagerInterface = mock(TaskCommunicatorManagerInterface.class);
    taskHeartbeatHandler = mock(TaskHeartbeatHandler.class);
    credentials = new Credentials();
    clock = new SystemClock();
    locationHint = TaskLocationHint.createTaskLocationHint(null, null);
    appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    dagId = TezDAGID.getInstance(appId, 1);
    vertexId = TezVertexID.getInstance(dagId, 1);
    appContext = mock(AppContext.class, RETURNS_DEEP_STUBS);
    when(appContext.getDAGRecoveryData()).thenReturn(null);
    appContext.setDAGRecoveryData(null);
    mockContainerId = mock(ContainerId.class);
    mockContainer = mock(Container.class);
    mockAMContainer = mock(AMContainer.class);
    when(mockAMContainer.getContainer()).thenReturn(mockContainer);
    when(mockContainer.getNodeHttpAddress()).thenReturn("localhost:1234");
    mockNodeId = mock(NodeId.class);
    mockHistoryHandler = mock(HistoryEventHandler.class);
    when(mockContainer.getId()).thenReturn(mockContainerId);
    when(mockContainer.getNodeId()).thenReturn(mockNodeId);
    when(mockAMContainer.getContainer()).thenReturn(mockContainer);
    when(appContext.getAllContainers().get(mockContainerId)).thenReturn(mockAMContainer);
    when(appContext.getHistoryHandler()).thenReturn(mockHistoryHandler);
    taskResource = Resource.newInstance(1024, 1);
    localResources = new HashMap<String, LocalResource>();
    environment = new HashMap<String, String>();
    javaOpts = "";
    leafVertex = false;
    containerContext = new ContainerContext(localResources, credentials, environment, javaOpts);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(conf)).when(vertex).getVertexConfig();
    eventHandler = new TestEventHandler();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, leafVertex, taskResource, containerContext, vertex);
    mockTaskSpec = mock(TaskSpec.class);
    mockVertex = mock(Vertex.class);
    ServicePluginInfo servicePluginInfo = new ServicePluginInfo().setContainerLauncherName(TezConstants.getTezYarnServicePluginName());
    when(mockVertex.getServicePluginInfo()).thenReturn(servicePluginInfo);
    when(mockVertex.getVertexConfig()).thenReturn(new VertexImpl.VertexConfigImpl(conf));
}

Example 24 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestTaskImpl method testFailedThenSpeculativeFailed.

@Test(timeout = 20000)
public void testFailedThenSpeculativeFailed() {
    conf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(conf)).when(vertex).getVertexConfig();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, leafVertex, taskResource, containerContext, vertex);
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(firstAttempt.getID());
    updateAttemptState(firstAttempt, TaskAttemptState.RUNNING);
    // Add a speculative task attempt
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl specAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(specAttempt.getID());
    updateAttemptState(specAttempt, TaskAttemptState.RUNNING);
    assertEquals(2, mockTask.getAttemptList().size());
    // Fail the first attempt
    updateAttemptState(firstAttempt, TaskAttemptState.FAILED);
    mockTask.handle(createTaskTAFailedEvent(firstAttempt.getID()));
    assertEquals(TaskState.FAILED, mockTask.getState());
    assertEquals(2, mockTask.getAttemptList().size());
    // Now fail the speculative attempt
    updateAttemptState(specAttempt, TaskAttemptState.FAILED);
    mockTask.handle(createTaskTAFailedEvent(specAttempt.getID()));
    assertEquals(TaskState.FAILED, mockTask.getState());
    assertEquals(2, mockTask.getAttemptList().size());
}

Example 25 with Vertex

use of org.apache.tez.dag.app.dag.Vertex in project tez by apache.

the class TestTaskImpl method testSucceededLeafTaskWithRetroFailures.

@SuppressWarnings("rawtypes")
@Test(timeout = 10000L)
public void testSucceededLeafTaskWithRetroFailures() throws InterruptedException {
    Configuration newConf = new Configuration(conf);
    newConf.setInt(TezConfiguration.TEZ_AM_TASK_MAX_FAILED_ATTEMPTS, 1);
    Vertex vertex = mock(Vertex.class);
    doReturn(new VertexImpl.VertexConfigImpl(newConf)).when(vertex).getVertexConfig();
    mockTask = new MockTaskImpl(vertexId, partition, eventHandler, conf, taskCommunicatorManagerInterface, clock, taskHeartbeatHandler, appContext, true, taskResource, containerContext, vertex);
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstMockTaskAttempt = mockTask.getAttemptList().get(0);
    launchTaskAttempt(firstMockTaskAttempt.getID());
    mockTask.handle(createTaskTAAddSpecAttempt(mockTask.getLastAttempt().getID()));
    MockTaskAttemptImpl secondMockTaskAttempt = mockTask.getAttemptList().get(1);
    launchTaskAttempt(secondMockTaskAttempt.getID());
    firstMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), 10, 10));
    secondMockTaskAttempt.handle(new TaskAttemptEventSchedule(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), 10, 10));
    firstMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventSubmitted(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), mockContainer.getId()));
    secondMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString())));
    firstMockTaskAttempt.handle(new TaskAttemptEventStartedRemotely(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString())));
    secondMockTaskAttempt.handle(new TaskAttemptEvent(TezTaskAttemptID.fromString(secondMockTaskAttempt.toString()), TaskAttemptEventType.TA_DONE));
    firstMockTaskAttempt.handle(new TaskAttemptEventAttemptFailed(TezTaskAttemptID.fromString(firstMockTaskAttempt.toString()), TaskAttemptEventType.TA_FAILED, TaskFailureType.NON_FATAL, "test", TaskAttemptTerminationCause.CONTAINER_EXITED));
    mockTask.handle(new TaskEventTASucceeded(secondMockTaskAttempt.getID()));
    firstMockTaskAttempt.handle(new TaskAttemptEventContainerTerminated(mockContainerId, firstMockTaskAttempt.getID(), "test", TaskAttemptTerminationCause.NO_PROGRESS));
    InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, 0);
    TezTaskAttemptID mockDestId = firstMockTaskAttempt.getID();
    EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId);
    TezEvent tzEvent = new TezEvent(mockReEvent, meta);
    TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, tzEvent, 1);
    firstMockTaskAttempt.handle(outputFailedEvent);
    mockTask.handle(new TaskEventTAFailed(firstMockTaskAttempt.getID(), TaskFailureType.NON_FATAL, mock(TaskAttemptEvent.class)));
    Assert.assertEquals(mockTask.getInternalState(), TaskStateInternal.SUCCEEDED);
}