Examples with FetchedInputAllocator org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator used on opensource projects

Example 1 with FetchedInputAllocator

use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.

the class TestShuffleManager method createShuffleManager.

private ShuffleManagerForTest createShuffleManager(InputContext inputContext, int expectedNumOfPhysicalInputs) throws IOException {
    Path outDirBase = new Path(".", "outDir");
    String[] outDirs = new String[] { outDirBase.toString() };
    doReturn(outDirs).when(inputContext).getWorkDirs();
    conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, inputContext.getWorkDirs());
    DataOutputBuffer out = new DataOutputBuffer();
    Token<JobTokenIdentifier> token = new Token<JobTokenIdentifier>(new JobTokenIdentifier(), new JobTokenSecretManager(null));
    token.write(out);
    doReturn(ByteBuffer.wrap(out.getData())).when(inputContext).getServiceConsumerMetaData(conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT));
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    return new ShuffleManagerForTest(inputContext, conf, expectedNumOfPhysicalInputs, 1024, false, -1, null, inputAllocator);
}

Example 2 with FetchedInputAllocator

use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents_WithOutOfOrderAttempts.

/**
 * In pipelined shuffle, check if processing & exceptions are done correctly when attempts are
 * received in out of order fashion (e.g attemptNum 1 arrives before attemptNum 0)
 *
 * @throws IOException
 */
@Test(timeout = 5000)
public void testPipelinedShuffleEvents_WithOutOfOrderAttempts() throws IOException {
    InputContext inputContext = createInputContext();
    ShuffleManager shuffleManager = createShuffleManager(inputContext);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    // 0--> 1 with spill id 0 (attemptNum 1).  attemptNum 0 is not sent.
    Event dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 1);
    handler.handleEvents(Collections.singletonList(dme));
    CompositeInputAttemptIdentifier expected = new CompositeInputAttemptIdentifier(1, 1, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1, 1);
    verify(shuffleManager, times(1)).addKnownInput(eq(HOST), eq(PORT), eq(expected), eq(0));
    // Let attemptNum 1 be scheduled.
    shuffleManager.shuffleInfoEventsMap.get(expected.getInputIdentifier()).scheduledForDownload = true;
    // Now send attemptNum 0.  This should throw exception, because attempt #1 is already added
    dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    verify(inputContext).killSelf(any(Throwable.class), anyString());
}

Example 3 with FetchedInputAllocator

use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents.

/**
 * In pipelined shuffle, check if multiple attempt numbers are processed and
 * exceptions are reported properly.
 *
 * @throws IOException
 */
@Test(timeout = 5000)
public void testPipelinedShuffleEvents() throws IOException {
    InputContext inputContext = createInputContext();
    ShuffleManager shuffleManager = createShuffleManager(inputContext);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    // 0--> 1 with spill id 0 (attemptNum 0)
    Event dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    CompositeInputAttemptIdentifier expectedId1 = new CompositeInputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0, 1);
    verify(shuffleManager, times(1)).addKnownInput(eq(HOST), eq(PORT), eq(expectedId1), eq(0));
    // 0--> 1 with spill id 1 (attemptNum 0)
    dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    CompositeInputAttemptIdentifier expectedId2 = new CompositeInputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1, 1);
    verify(shuffleManager, times(2)).addKnownInput(eq(HOST), eq(PORT), eq(expectedId2), eq(0));
    // Let attemptNum 0 be scheduled.
    shuffleManager.shuffleInfoEventsMap.get(expectedId2.getInputIdentifier()).scheduledForDownload = true;
    // 0--> 1 with spill id 1 (attemptNum 1).  This should report exception
    dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 1);
    handler.handleEvents(Collections.singletonList(dme));
    verify(inputContext).killSelf(any(Throwable.class), anyString());
}

Example 4 with FetchedInputAllocator

use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents_WithEmptyPartitions.

/**
 * In pipelined shuffle, check if processing & exceptions are done correctly when empty
 * partitions are sent
 *
 * @throws IOException
 */
@Test(timeout = 5000)
public void testPipelinedShuffleEvents_WithEmptyPartitions() throws IOException {
    InputContext inputContext = createInputContext();
    ShuffleManager shuffleManager = createShuffleManager(inputContext);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    // 0--> 1 with spill id 0 (attemptNum 0) with empty partitions
    BitSet bitSet = new BitSet(4);
    bitSet.flip(0, 4);
    Event dme = createDataMovementEvent(true, 0, 1, 0, false, bitSet, 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    InputAttemptIdentifier expected = new InputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0);
    verify(shuffleManager, times(1)).addCompletedInputWithNoData(expected);
    // 0--> 1 with spill id 1 (attemptNum 0)
    handler.handleEvents(Collections.singletonList(dme));
    dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 0);
    expected = new InputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1);
    verify(shuffleManager, times(2)).addCompletedInputWithNoData(expected);
    // Let attemptNum 0 be scheduled.
    shuffleManager.shuffleInfoEventsMap.get(expected.getInputIdentifier()).scheduledForDownload = true;
    // Now send attemptNum 1.  This should throw exception, because attempt #1 is already added
    dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 1);
    handler.handleEvents(Collections.singletonList(dme));
    verify(inputContext).killSelf(any(Throwable.class), anyString());
}

Example 5 with FetchedInputAllocator

use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testSimple.

@Test(timeout = 5000)
public void testSimple() throws IOException {
    InputContext inputContext = mock(InputContext.class);
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    int taskIndex = 1;
    Event dme = createDataMovementEvent(0, taskIndex, null);
    List<Event> eventList = new LinkedList<Event>();
    eventList.add(dme);
    handler.handleEvents(eventList);
    CompositeInputAttemptIdentifier expectedIdentifier = new CompositeInputAttemptIdentifier(taskIndex, 0, PATH_COMPONENT, 1);
    verify(shuffleManager).addKnownInput(eq(HOST), eq(PORT), eq(expectedIdentifier), eq(0));
}