Examples with DataMovementEvent org.apache.tez.runtime.api.events.DataMovementEvent used on opensource projects

Example 21 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestOutput method close.

@Override
public List<Event> close() throws Exception {
    LOG.info("Sending data movement event with value: " + output);
    getContext().getCounters().findCounter(COUNTER_NAME, COUNTER_NAME).increment(1);
    ;
    ByteBuffer result = ByteBuffer.allocate(4).putInt(output);
    result.flip();
    List<Event> events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs());
    for (int i = 0; i < getNumPhysicalOutputs(); i++) {
        DataMovementEvent event = DataMovementEvent.create(i, result);
        events.add(event);
    }
    ShuffleUserPayloads.VertexManagerEventPayloadProto.Builder vmBuilder = ShuffleUserPayloads.VertexManagerEventPayloadProto.newBuilder().setNumRecord(1);
    VertexManagerEvent vmEvent = VertexManagerEvent.create(getContext().getDestinationVertexName(), vmBuilder.build().toByteString().asReadOnlyByteBuffer());
    events.add(vmEvent);
    return events;
}

Example 22 with DataMovementEvent

use of org.apache.tez.runtime.api.events.DataMovementEvent in project tez by apache.

the class TestUnorderedKVOutput2 method testNonStartedOutput.

@Test(timeout = 5000)
public void testNonStartedOutput() throws Exception {
    OutputContext outputContext = OutputTestHelpers.createOutputContext();
    int numPartitions = 1;
    UnorderedKVOutput output = new UnorderedKVOutput(outputContext, numPartitions);
    output.initialize();
    List<Event> events = output.close();
    assertEquals(1, events.size());
    Event event1 = events.get(0);
    assertTrue(event1 instanceof DataMovementEvent);
    DataMovementEvent dme = (DataMovementEvent) event1;
    ByteBuffer bb = dme.getUserPayload();
    ShuffleUserPayloads.DataMovementEventPayloadProto shufflePayload = ShuffleUserPayloads.DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(bb));
    assertTrue(shufflePayload.hasEmptyPartitions());
    byte[] emptyPartitions = TezCommonUtils.decompressByteStringToByteArray(shufflePayload.getEmptyPartitions());
    BitSet emptyPartionsBitSet = TezUtilsInternal.fromByteArray(emptyPartitions);
    assertEquals(numPartitions, emptyPartionsBitSet.cardinality());
    for (int i = 0; i < numPartitions; i++) {
        assertTrue(emptyPartionsBitSet.get(i));
    }
}