Examples with VertexManagerEvent org.apache.tez.runtime.api.events.VertexManagerEvent used on opensource projects

Example 16 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class TestVertexImpl method testTez2684.

@Test(timeout = 5000)
public void testTez2684() throws IOException, TezException {
    setupPreDagCreation();
    dagPlan = createSamplerDAGPlan2();
    setupPostDagCreation();
    VertexImpl vA = vertices.get("A");
    VertexImpl vB = vertices.get("B");
    VertexImpl vC = vertices.get("C");
    // vA init & start
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_INIT));
    dispatcher.getEventHandler().handle(new VertexEvent(vA.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, vA.getState());
    Assert.assertEquals(VertexState.NEW, vB.getState());
    Assert.assertEquals(VertexState.NEW, vC.getState());
    // vB init
    dispatcher.getEventHandler().handle(new VertexEvent(vB.getVertexId(), VertexEventType.V_INIT));
    dispatcher.await();
    Assert.assertEquals(VertexState.INITED, vB.getState());
    Assert.assertEquals(VertexState.INITED, vC.getState());
    // Send VertexManagerEvent
    long[] sizes = new long[] { (100_000_000L) };
    Event vmEvent = getVertexManagerEvent(sizes, 1_060_000_000L, vB);
    sendTaskGeneratedEvent(vmEvent, EventProducerConsumerType.INPUT, vC, vB);
    Assert.assertEquals(VertexState.INITED, vC.getState());
    // vB start
    dispatcher.getEventHandler().handle(new VertexEvent(vB.getVertexId(), VertexEventType.V_START));
    dispatcher.await();
    Assert.assertEquals(VertexState.RUNNING, vC.getState());
}

Example 17 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class ShuffleUtils method generateVMEvent.

public static VertexManagerEvent generateVMEvent(OutputContext context, long[] sizePerPartition, boolean reportDetailedPartitionStats, Deflater deflater) throws IOException {
    ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder();
    long outputSize = context.getCounters().findCounter(TaskCounter.OUTPUT_BYTES).getValue();
    // Set this information only when required.  In pipelined shuffle,
    // multiple events would end up adding up to final output size.
    // This is needed for auto-reduce parallelism to work properly.
    vmBuilder.setOutputSize(outputSize);
    vmBuilder.setNumRecord(context.getCounters().findCounter(TaskCounter.OUTPUT_RECORDS).getValue() + context.getCounters().findCounter(TaskCounter.OUTPUT_LARGE_RECORDS).getValue());
    // set partition stats
    if (sizePerPartition != null && sizePerPartition.length > 0) {
        if (reportDetailedPartitionStats) {
            vmBuilder.setDetailedPartitionStats(getDetailedPartitionStatsForPhysicalOutput(sizePerPartition));
        } else {
            RoaringBitmap stats = getPartitionStatsForPhysicalOutput(sizePerPartition);
            DataOutputBuffer dout = new DataOutputBuffer();
            stats.serialize(dout);
            ByteString partitionStatsBytes = TezCommonUtils.compressByteArrayToByteString(dout.getData(), deflater);
            vmBuilder.setPartitionStats(partitionStatsBytes);
        }
    }
    VertexManagerEvent vmEvent = VertexManagerEvent.create(context.getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());
    return vmEvent;
}

Example 18 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class ShuffleUtils method generateEventsForNonStartedOutput.

/**
 * Generate events for outputs which have not been started.
 * @param eventList
 * @param numPhysicalOutputs
 * @param context
 * @param generateVmEvent whether to generate a vm event or not
 * @param isCompositeEvent whether to generate a CompositeDataMovementEvent or a DataMovementEvent
 * @param deflater
 * @throws IOException
 */
public static void generateEventsForNonStartedOutput(List<Event> eventList, int numPhysicalOutputs, OutputContext context, boolean generateVmEvent, boolean isCompositeEvent, Deflater deflater) throws IOException {
    DataMovementEventPayloadProto.Builder payloadBuilder = DataMovementEventPayloadProto.newBuilder();
    // Construct the VertexManager event if required.
    if (generateVmEvent) {
        ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder();
        vmBuilder.setOutputSize(0);
        VertexManagerEvent vmEvent = VertexManagerEvent.create(context.getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());
        eventList.add(vmEvent);
    }
    // Construct the DataMovementEvent
    // Always set empty partition information since no files were generated.
    LOG.info("Setting all {} partitions as empty for non-started output", numPhysicalOutputs);
    BitSet emptyPartitionDetails = new BitSet(numPhysicalOutputs);
    emptyPartitionDetails.set(0, numPhysicalOutputs, true);
    ByteString emptyPartitionsBytesString = TezCommonUtils.compressByteArrayToByteString(TezUtilsInternal.toByteArray(emptyPartitionDetails), deflater);
    payloadBuilder.setEmptyPartitions(emptyPartitionsBytesString);
    payloadBuilder.setRunDuration(0);
    DataMovementEventPayloadProto payloadProto = payloadBuilder.build();
    ByteBuffer dmePayload = payloadProto.toByteString().asReadOnlyByteBuffer();
    if (isCompositeEvent) {
        CompositeDataMovementEvent cdme = CompositeDataMovementEvent.create(0, numPhysicalOutputs, dmePayload);
        eventList.add(cdme);
    } else {
        DataMovementEvent dme = DataMovementEvent.create(0, dmePayload);
        eventList.add(dme);
    }
}

Example 19 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class ShuffleUtils method generateEventOnSpill.

/**
 * Generate events when spill happens
 *
 * @param eventList events would be added to this list
 * @param finalMergeEnabled
 * @param isLastEvent
 * @param context
 * @param spillId
 * @param spillRecord
 * @param numPhysicalOutputs
 * @param pathComponent
 * @param partitionStats
 * @param auxiliaryService
 * @throws IOException
 */
public static void generateEventOnSpill(List<Event> eventList, boolean finalMergeEnabled, boolean isLastEvent, OutputContext context, int spillId, TezSpillRecord spillRecord, int numPhysicalOutputs, boolean sendEmptyPartitionDetails, String pathComponent, @Nullable long[] partitionStats, boolean reportDetailedPartitionStats, String auxiliaryService, Deflater deflater) throws IOException {
    Preconditions.checkArgument(eventList != null, "EventList can't be null");
    context.notifyProgress();
    if (finalMergeEnabled) {
        Preconditions.checkArgument(isLastEvent, "Can not send multiple events when final merge is " + "enabled");
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("pathComponent=" + pathComponent + ", isLastEvent=" + isLastEvent + ", spillId=" + spillId + ", finalMergeDisabled=" + finalMergeEnabled + ", numPhysicalOutputs=" + numPhysicalOutputs);
    }
    ByteBuffer payload = generateDMEPayload(sendEmptyPartitionDetails, numPhysicalOutputs, spillRecord, context, spillId, finalMergeEnabled, isLastEvent, pathComponent, auxiliaryService, deflater);
    if (finalMergeEnabled || isLastEvent) {
        VertexManagerEvent vmEvent = generateVMEvent(context, partitionStats, reportDetailedPartitionStats, deflater);
        eventList.add(vmEvent);
    }
    CompositeDataMovementEvent csdme = CompositeDataMovementEvent.create(0, numPhysicalOutputs, payload);
    eventList.add(csdme);
}

Example 20 with VertexManagerEvent

use of org.apache.tez.runtime.api.events.VertexManagerEvent in project tez by apache.

the class TezEvent method serializeEvent.

private void serializeEvent(DataOutput out) throws IOException {
    if (event == null) {
        out.writeBoolean(false);
        return;
    }
    out.writeBoolean(true);
    out.writeInt(eventType.ordinal());
    out.writeLong(eventReceivedTime);
    if (eventType.equals(EventType.TASK_STATUS_UPDATE_EVENT)) {
        // TODO NEWTEZ convert to PB
        TaskStatusUpdateEvent sEvt = (TaskStatusUpdateEvent) event;
        sEvt.write(out);
    } else {
        AbstractMessage message;
        switch(eventType) {
            case CUSTOM_PROCESSOR_EVENT:
                message = ProtoConverters.convertCustomProcessorEventToProto((CustomProcessorEvent) event);
                break;
            case DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertDataMovementEventToProto((DataMovementEvent) event);
                break;
            case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeRoutedDataMovementEventToProto((CompositeRoutedDataMovementEvent) event);
                break;
            case COMPOSITE_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeDataMovementEventToProto((CompositeDataMovementEvent) event);
                break;
            case VERTEX_MANAGER_EVENT:
                message = ProtoConverters.convertVertexManagerEventToProto((VertexManagerEvent) event);
                break;
            case INPUT_READ_ERROR_EVENT:
                InputReadErrorEvent ideEvt = (InputReadErrorEvent) event;
                message = InputReadErrorEventProto.newBuilder().setIndex(ideEvt.getIndex()).setDiagnostics(ideEvt.getDiagnostics()).setVersion(ideEvt.getVersion()).build();
                break;
            case TASK_ATTEMPT_FAILED_EVENT:
                TaskAttemptFailedEvent tfEvt = (TaskAttemptFailedEvent) event;
                message = TaskAttemptFailedEventProto.newBuilder().setDiagnostics(tfEvt.getDiagnostics()).setTaskFailureType(TezConverterUtils.failureTypeToProto(tfEvt.getTaskFailureType())).build();
                break;
            case TASK_ATTEMPT_KILLED_EVENT:
                TaskAttemptKilledEvent tkEvent = (TaskAttemptKilledEvent) event;
                message = TaskAttemptKilledEventProto.newBuilder().setDiagnostics(tkEvent.getDiagnostics()).build();
                break;
            case TASK_ATTEMPT_COMPLETED_EVENT:
                message = TaskAttemptCompletedEventProto.newBuilder().build();
                break;
            case INPUT_FAILED_EVENT:
                InputFailedEvent ifEvt = (InputFailedEvent) event;
                message = InputFailedEventProto.newBuilder().setTargetIndex(ifEvt.getTargetIndex()).setVersion(ifEvt.getVersion()).build();
                break;
            case ROOT_INPUT_DATA_INFORMATION_EVENT:
                message = ProtoConverters.convertRootInputDataInformationEventToProto((InputDataInformationEvent) event);
                break;
            case ROOT_INPUT_INITIALIZER_EVENT:
                message = ProtoConverters.convertRootInputInitializerEventToProto((InputInitializerEvent) event);
                break;
            default:
                throw new TezUncheckedException("Unknown TezEvent" + ", type=" + eventType);
        }
        if (out instanceof OutputStream) {
            // DataOutputBuffer extends DataOutputStream
            int serializedSize = message.getSerializedSize();
            out.writeInt(serializedSize);
            int buffersize = serializedSize < CodedOutputStream.DEFAULT_BUFFER_SIZE ? serializedSize : CodedOutputStream.DEFAULT_BUFFER_SIZE;
            CodedOutputStream codedOut = CodedOutputStream.newInstance((OutputStream) out, buffersize);
            message.writeTo(codedOut);
            codedOut.flush();
        } else {
            byte[] eventBytes = message.toByteArray();
            out.writeInt(eventBytes.length);
            out.write(eventBytes);
        }
    }
}