Examples with InputFailedEvent org.apache.tez.runtime.api.events.InputFailedEvent used on opensource projects

Example 1 with InputFailedEvent

use of org.apache.tez.runtime.api.events.InputFailedEvent in project tez by apache.

the class Edge method sendDmEventOrIfEventToTasks.

void sendDmEventOrIfEventToTasks(TezEvent tezEvent, int srcTaskIndex, boolean isDataMovementEvent, Map<Integer, List<Integer>> taskAndInputIndices) {
    Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time");
    Event event = tezEvent.getEvent();
    // cache of event object per input index
    Map<Integer, TezEvent> inputIndicesWithEvents = Maps.newHashMap();
    for (Map.Entry<Integer, List<Integer>> entry : taskAndInputIndices.entrySet()) {
        int destTaskIndex = entry.getKey();
        List<Integer> inputIndices = entry.getValue();
        for (int i = 0; i < inputIndices.size(); ++i) {
            Integer inputIndex = inputIndices.get(i);
            TezEvent tezEventToSend = inputIndicesWithEvents.get(inputIndex);
            if (tezEventToSend == null) {
                Event e;
                if (isDataMovementEvent) {
                    DataMovementEvent dmEvent = (DataMovementEvent) event;
                    e = DataMovementEvent.create(dmEvent.getSourceIndex(), inputIndex, dmEvent.getVersion(), dmEvent.getUserPayload());
                } else {
                    InputFailedEvent ifEvent = ((InputFailedEvent) event);
                    e = InputFailedEvent.create(inputIndex, ifEvent.getVersion());
                }
                tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                // cache the event object per input because are unique per input index
                inputIndicesWithEvents.put(inputIndex, tezEventToSend);
            }
            Task destTask = destinationVertex.getTask(destTaskIndex);
            if (destTask == null) {
                throw new TezUncheckedException("Unexpected null task." + " sourceVertex=" + sourceVertex.getLogIdentifier() + " srcTaskIndex = " + srcTaskIndex + " destVertex=" + destinationVertex.getLogIdentifier() + " destTaskIndex=" + destTaskIndex + " destNumTasks=" + destinationVertex.getTotalTasks() + " edgeManager=" + edgeManager.getClass().getName());
            }
            sendEventToTask(destTask, tezEventToSend);
        }
    }
}

Example 2 with InputFailedEvent

use of org.apache.tez.runtime.api.events.InputFailedEvent in project tez by apache.

the class TestShuffleInputEventHandlerOrderedGrouped method testFailedEvent.

@Test(timeout = 5000)
public void testFailedEvent() throws IOException {
    List<Event> events = new LinkedList<Event>();
    int targetIdx = 1;
    InputFailedEvent failedEvent = InputFailedEvent.create(targetIdx, 0);
    events.add(failedEvent);
    handler.handleEvents(events);
    InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(targetIdx, 0);
    verify(scheduler).obsoleteInput(eq(expectedIdentifier));
}

Example 3 with InputFailedEvent

use of org.apache.tez.runtime.api.events.InputFailedEvent in project tez by apache.

the class TestShuffleInputEventHandlerOrderedGrouped method testPipelinedShuffle_WithObsoleteEvents.

@Test(timeout = 5000)
public void testPipelinedShuffle_WithObsoleteEvents() throws IOException, InterruptedException {
    // Process attempt #1 first
    int attemptNum = 1;
    int inputIdx = 1;
    Event dme1 = createDataMovementEvent(attemptNum, inputIdx, null, false, true, true, 0, attemptNum);
    handler.handleEvents(Collections.singletonList(dme1));
    CompositeInputAttemptIdentifier id1 = new CompositeInputAttemptIdentifier(inputIdx, attemptNum, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0, 1);
    verify(scheduler, times(1)).addKnownMapOutput(eq(HOST), eq(PORT), eq(1), eq(id1));
    assertTrue("Shuffle info events should not be empty for pipelined shuffle", !scheduler.pipelinedShuffleInfoEventsMap.isEmpty());
    int valuesInMapLocations = scheduler.mapLocations.values().size();
    assertTrue("Maplocations should have values. current size: " + valuesInMapLocations, valuesInMapLocations > 0);
    // start scheduling for download. Sets up scheduledForDownload in eventInfo.
    scheduler.getMapsForHost(scheduler.mapLocations.values().iterator().next());
    // send input failed event.
    List<Event> events = new LinkedList<Event>();
    int targetIdx = 1;
    InputFailedEvent failedEvent = InputFailedEvent.create(targetIdx, 0);
    events.add(failedEvent);
    handler.handleEvents(events);
    // task should issue kill request, as inputs are scheduled for download already.
    verify(scheduler, times(1)).killSelf(any(IOException.class), any(String.class));
}

Example 4 with InputFailedEvent

use of org.apache.tez.runtime.api.events.InputFailedEvent in project tez by apache.

the class TestDAGImpl method testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting.

@SuppressWarnings("unchecked")
@Test(timeout = 5000)
public void testEdgeManager_RouteInputSourceTaskFailedEventToDestinationLegacyRouting() {
    // Remove after legacy routing is removed
    setupDAGWithCustomEdge(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination, true);
    dispatcher.getEventHandler().handle(new DAGEvent(dagWithCustomEdge.getID(), DAGEventType.DAG_INIT));
    dispatcher.getEventHandler().handle(new DAGEventStartDag(dagWithCustomEdge.getID(), null));
    dispatcher.await();
    Assert.assertEquals(DAGState.RUNNING, dagWithCustomEdge.getState());
    VertexImpl v1 = (VertexImpl) dagWithCustomEdge.getVertex("vertex1");
    VertexImpl v2 = (VertexImpl) dagWithCustomEdge.getVertex("vertex2");
    dispatcher.await();
    Task t1 = v2.getTask(0);
    TaskAttemptImpl ta1 = (TaskAttemptImpl) t1.getAttempt(TezTaskAttemptID.getInstance(t1.getTaskId(), 0));
    InputFailedEvent ifEvent = InputFailedEvent.create(0, 1);
    TezEvent tezEvent = new TezEvent(ifEvent, new EventMetaData(EventProducerConsumerType.INPUT, "vertex1", "vertex2", ta1.getID()));
    dispatcher.getEventHandler().handle(new VertexEventRouteEvent(v2.getVertexId(), Lists.newArrayList(tezEvent)));
    dispatcher.await();
    v2.getTaskAttemptTezEvents(ta1.getID(), 0, 0, 1000);
    dispatcher.await();
    Assert.assertEquals(VertexState.FAILED, v2.getState());
    Assert.assertEquals(VertexState.KILLED, v1.getState());
    String diag = StringUtils.join(v2.getDiagnostics(), ",");
    Assert.assertTrue(diag.contains(ExceptionLocation.RouteInputSourceTaskFailedEventToDestination.name()));
}

Example 5 with InputFailedEvent

use of org.apache.tez.runtime.api.events.InputFailedEvent in project tez by apache.

the class ShuffleInputEventHandlerOrderedGrouped method handleEvent.

private void handleEvent(Event event) throws IOException {
    if (event instanceof DataMovementEvent) {
        numDmeEvents.incrementAndGet();
        DataMovementEvent dmEvent = (DataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(dmEvent.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        processDataMovementEvent(dmEvent, shufflePayload, emptyPartitionsBitSet);
        scheduler.updateEventReceivedTime();
    } else if (event instanceof CompositeRoutedDataMovementEvent) {
        CompositeRoutedDataMovementEvent crdme = (CompositeRoutedDataMovementEvent) event;
        DataMovementEventPayloadProto shufflePayload;
        try {
            shufflePayload = DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(crdme.getUserPayload()));
        } catch (InvalidProtocolBufferException e) {
            throw new TezUncheckedException("Unable to parse DataMovementEvent payload", e);
        }
        BitSet emptyPartitionsBitSet = null;
        if (shufflePayload.hasEmptyPartitions()) {
            try {
                byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions(), inflater);
                emptyPartitionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
            } catch (IOException e) {
                throw new TezUncheckedException("Unable to set the empty partition to succeeded", e);
            }
        }
        if (compositeFetch) {
            numDmeEvents.addAndGet(crdme.getCount());
            processCompositeRoutedDataMovementEvent(crdme, shufflePayload, emptyPartitionsBitSet);
        } else {
            for (int offset = 0; offset < crdme.getCount(); offset++) {
                numDmeEvents.incrementAndGet();
                processDataMovementEvent(crdme.expand(offset), shufflePayload, emptyPartitionsBitSet);
            }
        }
        scheduler.updateEventReceivedTime();
    } else if (event instanceof InputFailedEvent) {
        numObsoletionEvents.incrementAndGet();
        processTaskFailedEvent((InputFailedEvent) event);
    }
    if (numDmeEvents.get() + numObsoletionEvents.get() > nextToLogEventCount.get()) {
        logProgress(false);
        // Log every 50 events seen.
        nextToLogEventCount.addAndGet(50);
    }
}