Examples with TezEvent org.apache.tez.runtime.api.impl.TezEvent used on opensource projects

Example 66 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestEdge method testInvalidSourceTaskIndex.

@Test(timeout = 5000)
public void testInvalidSourceTaskIndex() throws Exception {
    EventHandler mockEventHandler = mock(EventHandler.class);
    Edge edge = new Edge(EdgeProperty.create(EdgeManagerPluginDescriptor.create(CustomEdgeManagerWithInvalidReturnValue.class.getName()).setUserPayload(new CustomEdgeManagerWithInvalidReturnValue.EdgeManagerConfig(1, 1, 1, -1).toUserPayload()), DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, OutputDescriptor.create(""), InputDescriptor.create("")), mockEventHandler, new TezConfiguration());
    TezVertexID v1Id = createVertexID(1);
    TezVertexID v2Id = createVertexID(2);
    edge.setSourceVertex(mockVertex("v1", v1Id, new LinkedHashMap<TezTaskID, Task>()));
    edge.setDestinationVertex(mockVertex("v2", v2Id, new LinkedHashMap<TezTaskID, Task>()));
    edge.initialize();
    try {
        TezEvent ireEvent = new TezEvent(InputReadErrorEvent.create("diag", 0, 1), new EventMetaData(EventProducerConsumerType.INPUT, "v2", "v1", TezTaskAttemptID.getInstance(TezTaskID.getInstance(v2Id, 1), 1)));
        edge.sendTezEventToSourceTasks(ireEvent);
        Assert.fail();
    } catch (AMUserCodeException e) {
        e.printStackTrace();
        assertTrue(e.getCause().getMessage().contains("SourceTaskIndex should not be negative"));
    }
}

Example 67 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestEdge method testCompositeEventHandling.

@SuppressWarnings({ "rawtypes" })
@Test(timeout = 5000)
public void testCompositeEventHandling() throws TezException {
    EventHandler eventHandler = mock(EventHandler.class);
    EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class), mock(InputDescriptor.class));
    Edge edge = new Edge(edgeProp, eventHandler, new TezConfiguration());
    TezVertexID srcVertexID = createVertexID(1);
    TezVertexID destVertexID = createVertexID(2);
    LinkedHashMap<TezTaskID, Task> srcTasks = mockTasks(srcVertexID, 1);
    LinkedHashMap<TezTaskID, Task> destTasks = mockTasks(destVertexID, 5);
    TezTaskID srcTaskID = srcTasks.keySet().iterator().next();
    Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks);
    Vertex destVertex = mockVertex("dest", destVertexID, destTasks);
    edge.setSourceVertex(srcVertex);
    edge.setDestinationVertex(destVertex);
    edge.initialize();
    // Task0, Attempt 0
    TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2);
    EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
    // Verification via a CompositeEvent
    CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(), ByteBuffer.wrap("bytes".getBytes()));
    // AttemptNum
    cdmEvent.setVersion(2);
    TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
    // Event setup to look like it would after the Vertex is done with it.
    edge.sendTezEventToDestinationTasks(tezEvent);
    verifyEvents(srcTAID, destTasks);
    // Same Verification via regular DataMovementEvents
    // Reset the mock
    resetTaskMocks(destTasks.values());
    for (int i = 0; i < destTasks.size(); i++) {
        DataMovementEvent dmEvent = DataMovementEvent.create(i, ByteBuffer.wrap("bytes".getBytes()));
        dmEvent.setVersion(2);
        tezEvent = new TezEvent(dmEvent, srcMeta);
        edge.sendTezEventToDestinationTasks(tezEvent);
    }
    verifyEvents(srcTAID, destTasks);
}

Example 68 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestTaskImpl method testSpeculatedThenRetroactiveFailure.

@Test(timeout = 20000)
public void testSpeculatedThenRetroactiveFailure() {
    TezTaskID taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    MockTaskAttemptImpl firstAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(firstAttempt.getID());
    updateAttemptState(firstAttempt, TaskAttemptState.RUNNING);
    // Add a speculative task attempt
    mockTask.handle(createTaskTAAddSpecAttempt(firstAttempt.getID()));
    MockTaskAttemptImpl specAttempt = mockTask.getLastAttempt();
    launchTaskAttempt(specAttempt.getID());
    updateAttemptState(specAttempt, TaskAttemptState.RUNNING);
    assertEquals(2, mockTask.getAttemptList().size());
    // Have the first task succeed
    eventHandler.events.clear();
    mockTask.handle(createTaskTASucceededEvent(firstAttempt.getID()));
    // The task should now have succeeded and sent kill to other attempt
    assertTaskSucceededState();
    verify(mockTask.stateChangeNotifier).taskSucceeded(any(String.class), eq(taskId), eq(firstAttempt.getID().getId()));
    @SuppressWarnings("rawtypes") Event event = eventHandler.events.get(eventHandler.events.size() - 1);
    assertEquals(TaskAttemptEventType.TA_KILL_REQUEST, event.getType());
    assertEquals(specAttempt.getID(), ((TaskAttemptEventKillRequest) event).getTaskAttemptID());
    // Emulate the spec attempt being killed
    mockTask.handle(createTaskTAKilledEvent(specAttempt.getID()));
    assertTaskSucceededState();
    // Now fail the attempt after it has succeeded
    TezTaskAttemptID mockDestId = mock(TezTaskAttemptID.class);
    TezEvent mockTezEvent = mock(TezEvent.class);
    EventMetaData meta = new EventMetaData(EventProducerConsumerType.INPUT, "Vertex", "Edge", mockDestId);
    when(mockTezEvent.getSourceInfo()).thenReturn(meta);
    TaskAttemptEventOutputFailed outputFailedEvent = new TaskAttemptEventOutputFailed(mockDestId, mockTezEvent, 1);
    eventHandler.events.clear();
    mockTask.handle(createTaskTAFailedEvent(firstAttempt.getID(), TaskFailureType.NON_FATAL, outputFailedEvent));
    // The task should still be in the scheduled state
    assertTaskScheduledState();
    event = eventHandler.events.get(eventHandler.events.size() - 1);
    Assert.assertEquals(VertexEventType.V_TASK_RESCHEDULED, event.getType());
    // There should be a new attempt, and report of output read error
    // should be the causal TA
    List<MockTaskAttemptImpl> attempts = mockTask.getAttemptList();
    Assert.assertEquals(3, attempts.size());
    MockTaskAttemptImpl newAttempt = attempts.get(2);
    Assert.assertEquals(mockDestId, newAttempt.getSchedulingCausalTA());
}

Example 69 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TestTaskCommunicatorManager1 method testTaskHeartbeatResponse.

@Test(timeout = 5000)
public void testTaskHeartbeatResponse() throws Exception {
    List<TezEvent> events = new ArrayList<TezEvent>();
    List<TezEvent> eventsToSend = new ArrayList<TezEvent>();
    TaskHeartbeatResponse response = generateHeartbeat(events, 0, 1, 2, eventsToSend);
    assertEquals(2, response.getNextFromEventId());
    assertEquals(eventsToSend, response.getEvents());
}

Example 70 with TezEvent

use of org.apache.tez.runtime.api.impl.TezEvent in project tez by apache.

the class TaskAttemptFinishedEvent method toProto.

public TaskAttemptFinishedProto toProto() throws IOException {
    TaskAttemptFinishedProto.Builder builder = TaskAttemptFinishedProto.newBuilder();
    builder.setTaskAttemptId(taskAttemptId.toString()).setState(state.ordinal()).setCreationTime(creationTime).setAllocationTime(allocationTime).setStartTime(startTime).setFinishTime(finishTime);
    if (taskFailureType != null) {
        builder.setTaskFailureType(TezConverterUtils.failureTypeToProto(taskFailureType));
    }
    if (creationCausalTA != null) {
        builder.setCreationCausalTA(creationCausalTA.toString());
    }
    if (diagnostics != null) {
        builder.setDiagnostics(diagnostics);
    }
    if (error != null) {
        builder.setErrorEnum(error.name());
    }
    if (tezCounters != null) {
        builder.setCounters(DagTypeConverters.convertTezCountersToProto(tezCounters));
    }
    if (dataEvents != null && !dataEvents.isEmpty()) {
        for (DataEventDependencyInfo info : dataEvents) {
            builder.addDataEvents(DataEventDependencyInfo.toProto(info));
        }
    }
    if (taGeneratedEvents != null && !taGeneratedEvents.isEmpty()) {
        for (TezEvent event : taGeneratedEvents) {
            builder.addTaGeneratedEvents(TezEventUtils.toProto(event));
        }
    }
    if (containerId != null) {
        builder.setContainerId(containerId.toString());
    }
    if (nodeId != null) {
        builder.setNodeId(nodeId.toString());
    }
    if (nodeHttpAddress != null) {
        builder.setNodeHttpAddress(nodeHttpAddress);
    }
    return builder.build();
}