Examples with CompositeEventRouteMetadata org.apache.tez.dag.api.EdgeManagerPluginOnDemand.CompositeEventRouteMetadata used on opensource projects

Example 6 with CompositeEventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.CompositeEventRouteMetadata in project tez by apache.

the class Edge method maybeAddTezEventForDestinationTask.

// return false is event could be routed but ran out of space in the list
public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, int srcTaskIndex, List<TezEvent> listToAdd, int listMaxSize, PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException {
    if (!routingNeeded) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
        }
        return true;
    } else {
        try {
            EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager;
            int taskIndex = attemptID.getTaskID().getId();
            switch(tezEvent.getEventType()) {
                case COMPOSITE_DATA_MOVEMENT_EVENT:
                    {
                        CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
                        CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand.routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex);
                        if (routeMeta != null) {
                            CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta);
                            TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                            tezEventToSend.setDestinationInfo(destinationMetaInfo);
                            listToAdd.add(tezEventToSend);
                        }
                    }
                    break;
                case INPUT_FAILED_EVENT:
                    {
                        InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                case DATA_MOVEMENT_EVENT:
                    {
                        DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
                        EventRouteMetadata routeMeta;
                        int numEventsDone;
                        if (pendingRoutes != null) {
                            routeMeta = pendingRoutes.getRouteMeta();
                            numEventsDone = pendingRoutes.getNumEventsRouted();
                        } else {
                            routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex);
                            numEventsDone = 0;
                        }
                        if (routeMeta != null) {
                            int listSize = listToAdd.size();
                            int numEvents = routeMeta.getNumEvents();
                            int[] targetIndices = routeMeta.getTargetIndices();
                            while (numEventsDone < numEvents && listSize++ < listMaxSize) {
                                DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
                                numEventsDone++;
                                TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
                                tezEventToSend.setDestinationInfo(destinationMetaInfo);
                                listToAdd.add(tezEventToSend);
                            }
                            if (numEventsDone < numEvents) {
                                pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
                                return false;
                            }
                        }
                    }
                    break;
                default:
                    throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
            }
        } catch (Exception e) {
            throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
        }
    }
    return true;
}

Example 7 with CompositeEventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.CompositeEventRouteMetadata in project tez by apache.

the class TestCartesianProductEdgeManagerPartitioned method testThreeWayV1.

private void testThreeWayV1(CartesianProductConfigProto config) throws Exception {
    when(mockContext.getSourceVertexName()).thenReturn("v1");
    when(mockContext.getSourceVertexNumTasks()).thenReturn(3);
    edgeManager.initialize(config);
    CompositeEventRouteMetadata compositeRoutingData = edgeManager.routeCompositeDataMovementEventToDestination(1, 1);
    assertNotNull(compositeRoutingData);
    assertEquals(1, compositeRoutingData.getCount());
    assertEquals(0, compositeRoutingData.getSource());
    assertEquals(1, compositeRoutingData.getTarget());
    EventRouteMetadata routingData = edgeManager.routeInputSourceTaskFailedEventToDestination(1, 1);
    assertNotNull(routingData);
    assertEquals(1, routingData.getNumEvents());
    assertArrayEquals(new int[] { 1 }, routingData.getTargetIndices());
    assertEquals(2, edgeManager.routeInputErrorEventToSource(1, 2));
    assertEquals(24, edgeManager.getNumDestinationConsumerTasks(1));
    assertEquals(3, edgeManager.getNumDestinationTaskPhysicalInputs(10));
    assertEquals(3, edgeManager.getNumSourceTaskPhysicalOutputs(2));
}

Example 8 with CompositeEventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.CompositeEventRouteMetadata in project tez by apache.

the class TestCartesianProductEdgeManagerPartitioned method testThreeWayV0.

private void testThreeWayV0(CartesianProductConfigProto config) throws Exception {
    when(mockContext.getSourceVertexName()).thenReturn("v0");
    when(mockContext.getSourceVertexNumTasks()).thenReturn(2);
    edgeManager.initialize(config);
    CompositeEventRouteMetadata compositeRoutingData = edgeManager.routeCompositeDataMovementEventToDestination(1, 1);
    assertNotNull(compositeRoutingData);
    assertEquals(1, compositeRoutingData.getCount());
    assertEquals(0, compositeRoutingData.getSource());
    assertEquals(1, compositeRoutingData.getTarget());
    EventRouteMetadata routingData = edgeManager.routeInputSourceTaskFailedEventToDestination(1, 1);
    assertNotNull(routingData);
    assertEquals(1, routingData.getNumEvents());
    assertArrayEquals(new int[] { 1 }, routingData.getTargetIndices());
    assertEquals(1, edgeManager.routeInputErrorEventToSource(1, 1));
    assertEquals(24, edgeManager.getNumDestinationConsumerTasks(1));
    assertEquals(2, edgeManager.getNumDestinationTaskPhysicalInputs(10));
    assertEquals(4, edgeManager.getNumSourceTaskPhysicalOutputs(2));
}

Example 9 with CompositeEventRouteMetadata

use of org.apache.tez.dag.api.EdgeManagerPluginOnDemand.CompositeEventRouteMetadata in project tez by apache.

the class TestCartesianProductEdgeManagerPartitioned method testTwoWayV0.

private void testTwoWayV0(CartesianProductConfigProto config) throws Exception {
    when(mockContext.getSourceVertexName()).thenReturn("v0");
    when(mockContext.getSourceVertexNumTasks()).thenReturn(2);
    edgeManager.initialize(config);
    CompositeEventRouteMetadata compositeRoutingData = edgeManager.routeCompositeDataMovementEventToDestination(1, 1);
    assertNotNull(compositeRoutingData);
    assertEquals(1, compositeRoutingData.getCount());
    assertEquals(0, compositeRoutingData.getSource());
    assertEquals(1, compositeRoutingData.getTarget());
    EventRouteMetadata routingData = edgeManager.routeDataMovementEventToDestination(1, 0, 1);
    assertNotNull(routingData);
    assertEquals(1, routingData.getNumEvents());
    assertArrayEquals(new int[] { 1 }, routingData.getTargetIndices());
    routingData = edgeManager.routeDataMovementEventToDestination(1, 1, 1);
    assertNull(routingData);
    routingData = edgeManager.routeInputSourceTaskFailedEventToDestination(1, 1);
    assertNotNull(routingData);
    assertEquals(1, routingData.getNumEvents());
    assertArrayEquals(new int[] { 1 }, routingData.getTargetIndices());
    assertEquals(1, edgeManager.routeInputErrorEventToSource(1, 1));
    assertEquals(12, edgeManager.getNumDestinationConsumerTasks(1));
    assertEquals(2, edgeManager.getNumDestinationTaskPhysicalInputs(10));
    assertEquals(3, edgeManager.getNumSourceTaskPhysicalOutputs(2));
}