use of org.apache.tez.runtime.api.events.CompositeDataMovementEvent in project tez by apache.
the class TestEdge method testCompositeEventHandling.
@SuppressWarnings({ "rawtypes" })
@Test(timeout = 5000)
public void testCompositeEventHandling() throws TezException {
EventHandler eventHandler = mock(EventHandler.class);
EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class), mock(InputDescriptor.class));
Edge edge = new Edge(edgeProp, eventHandler, new TezConfiguration());
TezVertexID srcVertexID = createVertexID(1);
TezVertexID destVertexID = createVertexID(2);
LinkedHashMap<TezTaskID, Task> srcTasks = mockTasks(srcVertexID, 1);
LinkedHashMap<TezTaskID, Task> destTasks = mockTasks(destVertexID, 5);
TezTaskID srcTaskID = srcTasks.keySet().iterator().next();
Vertex srcVertex = mockVertex("src", srcVertexID, srcTasks);
Vertex destVertex = mockVertex("dest", destVertexID, destTasks);
edge.setSourceVertex(srcVertex);
edge.setDestinationVertex(destVertex);
edge.initialize();
// Task0, Attempt 0
TezTaskAttemptID srcTAID = createTAIDForTest(srcTaskID, 2);
EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
// Verification via a CompositeEvent
CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(), ByteBuffer.wrap("bytes".getBytes()));
// AttemptNum
cdmEvent.setVersion(2);
TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
// Event setup to look like it would after the Vertex is done with it.
edge.sendTezEventToDestinationTasks(tezEvent);
verifyEvents(srcTAID, destTasks);
// Same Verification via regular DataMovementEvents
// Reset the mock
resetTaskMocks(destTasks.values());
for (int i = 0; i < destTasks.size(); i++) {
DataMovementEvent dmEvent = DataMovementEvent.create(i, ByteBuffer.wrap("bytes".getBytes()));
dmEvent.setVersion(2);
tezEvent = new TezEvent(dmEvent, srcMeta);
edge.sendTezEventToDestinationTasks(tezEvent);
}
verifyEvents(srcTAID, destTasks);
}
use of org.apache.tez.runtime.api.events.CompositeDataMovementEvent in project tez by apache.
the class ShuffleUtils method generateEventsForNonStartedOutput.
/**
* Generate events for outputs which have not been started.
* @param eventList
* @param numPhysicalOutputs
* @param context
* @param generateVmEvent whether to generate a vm event or not
* @param isCompositeEvent whether to generate a CompositeDataMovementEvent or a DataMovementEvent
* @param deflater
* @throws IOException
*/
public static void generateEventsForNonStartedOutput(List<Event> eventList, int numPhysicalOutputs, OutputContext context, boolean generateVmEvent, boolean isCompositeEvent, Deflater deflater) throws IOException {
DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto.newBuilder();
// Construct the VertexManager event if required.
if (generateVmEvent) {
ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder();
vmBuilder.setOutputSize(0);
VertexManagerEvent vmEvent = VertexManagerEvent.create(context.getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());
eventList.add(vmEvent);
}
// Construct the DataMovementEvent
// Always set empty partition information since no files were generated.
LOG.info("Setting all {} partitions as empty for non-started output", numPhysicalOutputs);
BitSet emptyPartitionDetails = new BitSet(numPhysicalOutputs);
emptyPartitionDetails.set(0, numPhysicalOutputs, true);
ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString(TezUtilsInternal.toByteArray(emptyPartitionDetails), deflater);
payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
payloadBuilder.setRunDuration(0);
DataMovementEventPayloadProto payloadProto = payloadBuilder.build();
ByteBuffer dmePayload = payloadProto.toByteString().asReadOnlyByteBuffer();
if (isCompositeEvent) {
CompositeDataMovementEvent cdme = CompositeDataMovementEvent.create(0, numPhysicalOutputs, dmePayload);
eventList.add(cdme);
} else {
DataMovementEvent dme = DataMovementEvent.create(0, dmePayload);
eventList.add(dme);
}
}
use of org.apache.tez.runtime.api.events.CompositeDataMovementEvent in project tez by apache.
the class ShuffleUtils method generateEventOnSpill.
/**
* Generate events when spill happens
*
* @param eventList events would be added to this list
* @param finalMergeEnabled
* @param isLastEvent
* @param context
* @param spillId
* @param spillRecord
* @param numPhysicalOutputs
* @param pathComponent
* @param partitionStats
* @param auxiliaryService
* @throws IOException
*/
public static void generateEventOnSpill(List<Event> eventList, boolean finalMergeEnabled, boolean isLastEvent, OutputContext context, int spillId, TezSpillRecord spillRecord, int numPhysicalOutputs, boolean sendEmptyPartitionDetails, String pathComponent, @Nullable long[] partitionStats, boolean reportDetailedPartitionStats, String auxiliaryService, Deflater deflater) throws IOException {
Preconditions.checkArgument(eventList != null, "EventList can't be null");
context.notifyProgress();
if (finalMergeEnabled) {
Preconditions.checkArgument(isLastEvent, "Can not send multiple events when final merge is " + "enabled");
}
if (LOG.isDebugEnabled()) {
LOG.debug("pathComponent=" + pathComponent + ", isLastEvent=" + isLastEvent + ", spillId=" + spillId + ", finalMergeDisabled=" + finalMergeEnabled + ", numPhysicalOutputs=" + numPhysicalOutputs);
}
ByteBuffer payload = generateDMEPayload(sendEmptyPartitionDetails, numPhysicalOutputs, spillRecord, context, spillId, finalMergeEnabled, isLastEvent, pathComponent, auxiliaryService, deflater);
if (finalMergeEnabled || isLastEvent) {
VertexManagerEvent vmEvent = generateVMEvent(context, partitionStats, reportDetailedPartitionStats, deflater);
eventList.add(vmEvent);
}
CompositeDataMovementEvent csdme = CompositeDataMovementEvent.create(0, numPhysicalOutputs, payload);
eventList.add(csdme);
}
use of org.apache.tez.runtime.api.events.CompositeDataMovementEvent in project tez by apache.
the class TezEvent method serializeEvent.
private void serializeEvent(DataOutput out) throws IOException {
if (event == null) {
out.writeBoolean(false);
return;
}
out.writeBoolean(true);
out.writeInt(eventType.ordinal());
out.writeLong(eventReceivedTime);
if (eventType.equals(EventType.TASK_STATUS_UPDATE_EVENT)) {
// TODO NEWTEZ convert to PB
TaskStatusUpdateEvent sEvt = (TaskStatusUpdateEvent) event;
sEvt.write(out);
} else {
AbstractMessage message;
switch(eventType) {
case CUSTOM_PROCESSOR_EVENT:
message = ProtoConverters.convertCustomProcessorEventToProto((CustomProcessorEvent) event);
break;
case DATA_MOVEMENT_EVENT:
message = ProtoConverters.convertDataMovementEventToProto((DataMovementEvent) event);
break;
case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT:
message = ProtoConverters.convertCompositeRoutedDataMovementEventToProto((CompositeRoutedDataMovementEvent) event);
break;
case COMPOSITE_DATA_MOVEMENT_EVENT:
message = ProtoConverters.convertCompositeDataMovementEventToProto((CompositeDataMovementEvent) event);
break;
case VERTEX_MANAGER_EVENT:
message = ProtoConverters.convertVertexManagerEventToProto((VertexManagerEvent) event);
break;
case INPUT_READ_ERROR_EVENT:
InputReadErrorEvent ideEvt = (InputReadErrorEvent) event;
message = InputReadErrorEventProto.newBuilder().setIndex(ideEvt.getIndex()).setDiagnostics(ideEvt.getDiagnostics()).setVersion(ideEvt.getVersion()).build();
break;
case TASK_ATTEMPT_FAILED_EVENT:
TaskAttemptFailedEvent tfEvt = (TaskAttemptFailedEvent) event;
message = TaskAttemptFailedEventProto.newBuilder().setDiagnostics(tfEvt.getDiagnostics()).setTaskFailureType(TezConverterUtils.failureTypeToProto(tfEvt.getTaskFailureType())).build();
break;
case TASK_ATTEMPT_KILLED_EVENT:
TaskAttemptKilledEvent tkEvent = (TaskAttemptKilledEvent) event;
message = TaskAttemptKilledEventProto.newBuilder().setDiagnostics(tkEvent.getDiagnostics()).build();
break;
case TASK_ATTEMPT_COMPLETED_EVENT:
message = TaskAttemptCompletedEventProto.newBuilder().build();
break;
case INPUT_FAILED_EVENT:
InputFailedEvent ifEvt = (InputFailedEvent) event;
message = InputFailedEventProto.newBuilder().setTargetIndex(ifEvt.getTargetIndex()).setVersion(ifEvt.getVersion()).build();
break;
case ROOT_INPUT_DATA_INFORMATION_EVENT:
message = ProtoConverters.convertRootInputDataInformationEventToProto((InputDataInformationEvent) event);
break;
case ROOT_INPUT_INITIALIZER_EVENT:
message = ProtoConverters.convertRootInputInitializerEventToProto((InputInitializerEvent) event);
break;
default:
throw new TezUncheckedException("Unknown TezEvent" + ", type=" + eventType);
}
if (out instanceof OutputStream) {
// DataOutputBuffer extends DataOutputStream
int serializedSize = message.getSerializedSize();
out.writeInt(serializedSize);
int buffersize = serializedSize < CodedOutputStream.DEFAULT_BUFFER_SIZE ? serializedSize : CodedOutputStream.DEFAULT_BUFFER_SIZE;
CodedOutputStream codedOut = CodedOutputStream.newInstance((OutputStream) out, buffersize);
message.writeTo(codedOut);
codedOut.flush();
} else {
byte[] eventBytes = message.toByteArray();
out.writeInt(eventBytes.length);
out.write(eventBytes);
}
}
}
use of org.apache.tez.runtime.api.events.CompositeDataMovementEvent in project tez by apache.
the class Edge method maybeAddTezEventForDestinationTask.
// return false is event could be routed but ran out of space in the list
public boolean maybeAddTezEventForDestinationTask(TezEvent tezEvent, TezTaskAttemptID attemptID, int srcTaskIndex, List<TezEvent> listToAdd, int listMaxSize, PendingEventRouteMetadata pendingRoutes) throws AMUserCodeException {
if (!routingNeeded) {
if (LOG.isDebugEnabled()) {
LOG.debug("Not routing events since destination vertex has 0 tasks" + generateCommonDebugString(srcTaskIndex, tezEvent));
}
return true;
} else {
try {
EdgeManagerPluginOnDemand edgeManagerOnDemand = (EdgeManagerPluginOnDemand) edgeManager;
int taskIndex = attemptID.getTaskID().getId();
switch(tezEvent.getEventType()) {
case COMPOSITE_DATA_MOVEMENT_EVENT:
{
CompositeDataMovementEvent compEvent = (CompositeDataMovementEvent) tezEvent.getEvent();
CompositeEventRouteMetadata routeMeta = edgeManagerOnDemand.routeCompositeDataMovementEventToDestination(srcTaskIndex, taskIndex);
if (routeMeta != null) {
CompositeRoutedDataMovementEvent edme = compEvent.expandRouted(routeMeta);
TezEvent tezEventToSend = new TezEvent(edme, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
tezEventToSend.setDestinationInfo(destinationMetaInfo);
listToAdd.add(tezEventToSend);
}
}
break;
case INPUT_FAILED_EVENT:
{
InputFailedEvent ifEvent = (InputFailedEvent) tezEvent.getEvent();
EventRouteMetadata routeMeta;
int numEventsDone;
if (pendingRoutes != null) {
routeMeta = pendingRoutes.getRouteMeta();
numEventsDone = pendingRoutes.getNumEventsRouted();
} else {
routeMeta = edgeManagerOnDemand.routeInputSourceTaskFailedEventToDestination(srcTaskIndex, taskIndex);
numEventsDone = 0;
}
if (routeMeta != null) {
int listSize = listToAdd.size();
int numEvents = routeMeta.getNumEvents();
int[] targetIndices = routeMeta.getTargetIndices();
while (numEventsDone < numEvents && listSize++ < listMaxSize) {
InputFailedEvent e = ifEvent.makeCopy(targetIndices[numEventsDone]);
numEventsDone++;
TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
tezEventToSend.setDestinationInfo(destinationMetaInfo);
listToAdd.add(tezEventToSend);
}
if (numEventsDone < numEvents) {
pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
return false;
}
}
}
break;
case DATA_MOVEMENT_EVENT:
{
DataMovementEvent dmEvent = (DataMovementEvent) tezEvent.getEvent();
EventRouteMetadata routeMeta;
int numEventsDone;
if (pendingRoutes != null) {
routeMeta = pendingRoutes.getRouteMeta();
numEventsDone = pendingRoutes.getNumEventsRouted();
} else {
routeMeta = edgeManagerOnDemand.routeDataMovementEventToDestination(srcTaskIndex, dmEvent.getSourceIndex(), taskIndex);
numEventsDone = 0;
}
if (routeMeta != null) {
int listSize = listToAdd.size();
int numEvents = routeMeta.getNumEvents();
int[] targetIndices = routeMeta.getTargetIndices();
while (numEventsDone < numEvents && listSize++ < listMaxSize) {
DataMovementEvent e = dmEvent.makeCopy(targetIndices[numEventsDone]);
numEventsDone++;
TezEvent tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo(), tezEvent.getEventReceivedTime());
tezEventToSend.setDestinationInfo(destinationMetaInfo);
listToAdd.add(tezEventToSend);
}
if (numEventsDone < numEvents) {
pendingEvents.put(attemptID, new PendingEventRouteMetadata(routeMeta, tezEvent, numEventsDone));
return false;
}
}
}
break;
default:
throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
}
} catch (Exception e) {
throw new AMUserCodeException(Source.EdgeManager, "Fail to maybeAddTezEventForDestinationTask, event:" + tezEvent.getEvent() + ", sourceInfo:" + tezEvent.getSourceInfo() + ", destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
}
}
return true;
}
Aggregations