Examples with CustomProcessorEvent org.apache.tez.runtime.api.events.CustomProcessorEvent used on opensource projects

Example 1 with CustomProcessorEvent

use of org.apache.tez.runtime.api.events.CustomProcessorEvent in project hive by apache.

the class TezProcessor method handleEvents.

@Override
public void handleEvents(List<Event> arg0) {
    // As of now only used for Bucket MapJoin, there is exactly one event in the list.
    assert arg0.size() <= 1;
    for (Event event : arg0) {
        CustomProcessorEvent cpEvent = (CustomProcessorEvent) event;
        ByteBuffer buffer = cpEvent.getPayload();
        // Get int view of the buffer
        IntBuffer intBuffer = buffer.asIntBuffer();
        jobConf.setInt(Constants.LLAP_NUM_BUCKETS, intBuffer.get(0));
        jobConf.setInt(Constants.LLAP_BUCKET_ID, intBuffer.get(1));
    }
}

Example 2 with CustomProcessorEvent

use of org.apache.tez.runtime.api.events.CustomProcessorEvent in project tez by apache.

the class TestVertexManager method testSendCustomProcessorEvent.

@Test(timeout = 5000)
public void testSendCustomProcessorEvent() throws Exception {
    VertexManager vm = new VertexManager(VertexManagerPluginDescriptor.create(CustomVertexManager.class.getName()), UserGroupInformation.getCurrentUser(), mockVertex, mockAppContext, mock(StateChangeNotifier.class));
    ArgumentCaptor<VertexEventRouteEvent> requestCaptor = ArgumentCaptor.forClass(VertexEventRouteEvent.class);
    when(mockVertex.getTotalTasks()).thenReturn(2);
    List<CustomProcessorEvent> events = new ArrayList<>();
    // task id too small, should fail
    try {
        vm.pluginContext.sendEventToProcessor(events, -1);
        fail("Should fail for invalid task id");
    } catch (IllegalArgumentException exception) {
        assertTrue(exception.getMessage().contains("Invalid taskId"));
    }
    // task id too large, should fail
    try {
        vm.pluginContext.sendEventToProcessor(events, 10);
        fail("Should fail for invalid task id");
    } catch (IllegalArgumentException exception) {
        assertTrue(exception.getMessage().contains("Invalid taskId"));
    }
    // null event, do nothing
    vm.pluginContext.sendEventToProcessor(null, 0);
    verify(mockHandler, never()).handle(requestCaptor.capture());
    // empty event
    vm.pluginContext.sendEventToProcessor(events, 1);
    verify(mockHandler, never()).handle(requestCaptor.capture());
    // events.add();
    byte[] payload = new byte[] { 1, 2, 3 };
    events.add(CustomProcessorEvent.create(ByteBuffer.wrap(payload)));
    vm.pluginContext.sendEventToProcessor(events, 1);
    verify(mockHandler, times(1)).handle(requestCaptor.capture());
    CustomProcessorEvent cpe = (CustomProcessorEvent) (requestCaptor.getValue().getEvents().get(0).getEvent());
    // should be able to get payload any times
    for (int i = 0; i < 2; i++) {
        ByteBuffer payloadBuffer = cpe.getPayload();
        assertEquals(payload.length, payloadBuffer.remaining());
        for (byte aPayload : payload) {
            assertEquals(aPayload, payloadBuffer.get());
        }
    }
}

Example 3 with CustomProcessorEvent

use of org.apache.tez.runtime.api.events.CustomProcessorEvent in project tez by apache.

the class TezEvent method serializeEvent.

private void serializeEvent(DataOutput out) throws IOException {
    if (event == null) {
        out.writeBoolean(false);
        return;
    }
    out.writeBoolean(true);
    out.writeInt(eventType.ordinal());
    out.writeLong(eventReceivedTime);
    if (eventType.equals(EventType.TASK_STATUS_UPDATE_EVENT)) {
        // TODO NEWTEZ convert to PB
        TaskStatusUpdateEvent sEvt = (TaskStatusUpdateEvent) event;
        sEvt.write(out);
    } else {
        AbstractMessage message;
        switch(eventType) {
            case CUSTOM_PROCESSOR_EVENT:
                message = ProtoConverters.convertCustomProcessorEventToProto((CustomProcessorEvent) event);
                break;
            case DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertDataMovementEventToProto((DataMovementEvent) event);
                break;
            case COMPOSITE_ROUTED_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeRoutedDataMovementEventToProto((CompositeRoutedDataMovementEvent) event);
                break;
            case COMPOSITE_DATA_MOVEMENT_EVENT:
                message = ProtoConverters.convertCompositeDataMovementEventToProto((CompositeDataMovementEvent) event);
                break;
            case VERTEX_MANAGER_EVENT:
                message = ProtoConverters.convertVertexManagerEventToProto((VertexManagerEvent) event);
                break;
            case INPUT_READ_ERROR_EVENT:
                InputReadErrorEvent ideEvt = (InputReadErrorEvent) event;
                message = InputReadErrorEventProto.newBuilder().setIndex(ideEvt.getIndex()).setDiagnostics(ideEvt.getDiagnostics()).setVersion(ideEvt.getVersion()).setIsLocalFetch(ideEvt.isLocalFetch()).setIsDiskErrorAtSource(ideEvt.isDiskErrorAtSource()).setDestinationLocalhostName(ideEvt.getDestinationLocalhostName()).build();
                break;
            case TASK_ATTEMPT_FAILED_EVENT:
                TaskAttemptFailedEvent tfEvt = (TaskAttemptFailedEvent) event;
                message = TaskAttemptFailedEventProto.newBuilder().setDiagnostics(tfEvt.getDiagnostics()).setTaskFailureType(TezConverterUtils.failureTypeToProto(tfEvt.getTaskFailureType())).build();
                break;
            case TASK_ATTEMPT_KILLED_EVENT:
                TaskAttemptKilledEvent tkEvent = (TaskAttemptKilledEvent) event;
                message = TaskAttemptKilledEventProto.newBuilder().setDiagnostics(tkEvent.getDiagnostics()).build();
                break;
            case TASK_ATTEMPT_COMPLETED_EVENT:
                message = TaskAttemptCompletedEventProto.newBuilder().build();
                break;
            case INPUT_FAILED_EVENT:
                InputFailedEvent ifEvt = (InputFailedEvent) event;
                message = InputFailedEventProto.newBuilder().setTargetIndex(ifEvt.getTargetIndex()).setVersion(ifEvt.getVersion()).build();
                break;
            case ROOT_INPUT_DATA_INFORMATION_EVENT:
                message = ProtoConverters.convertRootInputDataInformationEventToProto((InputDataInformationEvent) event);
                break;
            case ROOT_INPUT_INITIALIZER_EVENT:
                message = ProtoConverters.convertRootInputInitializerEventToProto((InputInitializerEvent) event);
                break;
            default:
                throw new TezUncheckedException("Unknown TezEvent" + ", type=" + eventType);
        }
        if (out instanceof OutputStream) {
            // DataOutputBuffer extends DataOutputStream
            int serializedSize = message.getSerializedSize();
            out.writeInt(serializedSize);
            int buffersize = serializedSize < CodedOutputStream.DEFAULT_BUFFER_SIZE ? serializedSize : CodedOutputStream.DEFAULT_BUFFER_SIZE;
            CodedOutputStream codedOut = CodedOutputStream.newInstance((OutputStream) out, buffersize);
            message.writeTo(codedOut);
            codedOut.flush();
        } else {
            byte[] eventBytes = message.toByteArray();
            out.writeInt(eventBytes.length);
            out.write(eventBytes);
        }
    }
}