use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.
the class TestShuffleManager method createShuffleManager.
private ShuffleManagerForTest createShuffleManager(InputContext inputContext, int expectedNumOfPhysicalInputs) throws IOException {
Path outDirBase = new Path(".", "outDir");
String[] outDirs = new String[] { outDirBase.toString() };
doReturn(outDirs).when(inputContext).getWorkDirs();
conf.setStrings(TezRuntimeFrameworkConfigs.LOCAL_DIRS, inputContext.getWorkDirs());
DataOutputBuffer out = new DataOutputBuffer();
Token<JobTokenIdentifier> token = new Token<JobTokenIdentifier>(new JobTokenIdentifier(), new JobTokenSecretManager(null));
token.write(out);
doReturn(ByteBuffer.wrap(out.getData())).when(inputContext).getServiceConsumerMetaData(conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT));
FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
return new ShuffleManagerForTest(inputContext, conf, expectedNumOfPhysicalInputs, 1024, false, -1, null, inputAllocator);
}
use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.
the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents_WithOutOfOrderAttempts.
/**
* In pipelined shuffle, check if processing & exceptions are done correctly when attempts are
* received in out of order fashion (e.g attemptNum 1 arrives before attemptNum 0)
*
* @throws IOException
*/
@Test(timeout = 5000)
public void testPipelinedShuffleEvents_WithOutOfOrderAttempts() throws IOException {
InputContext inputContext = createInputContext();
ShuffleManager shuffleManager = createShuffleManager(inputContext);
FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
// 0--> 1 with spill id 0 (attemptNum 1). attemptNum 0 is not sent.
Event dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 1);
handler.handleEvents(Collections.singletonList(dme));
CompositeInputAttemptIdentifier expected = new CompositeInputAttemptIdentifier(1, 1, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1, 1);
verify(shuffleManager, times(1)).addKnownInput(eq(HOST), eq(PORT), eq(expected), eq(0));
// Let attemptNum 1 be scheduled.
shuffleManager.shuffleInfoEventsMap.get(expected.getInputIdentifier()).scheduledForDownload = true;
// Now send attemptNum 0. This should throw exception, because attempt #1 is already added
dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 0);
handler.handleEvents(Collections.singletonList(dme));
verify(inputContext).killSelf(any(Throwable.class), anyString());
}
use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.
the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents.
/**
* In pipelined shuffle, check if multiple attempt numbers are processed and
* exceptions are reported properly.
*
* @throws IOException
*/
@Test(timeout = 5000)
public void testPipelinedShuffleEvents() throws IOException {
InputContext inputContext = createInputContext();
ShuffleManager shuffleManager = createShuffleManager(inputContext);
FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
// 0--> 1 with spill id 0 (attemptNum 0)
Event dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 0);
handler.handleEvents(Collections.singletonList(dme));
CompositeInputAttemptIdentifier expectedId1 = new CompositeInputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0, 1);
verify(shuffleManager, times(1)).addKnownInput(eq(HOST), eq(PORT), eq(expectedId1), eq(0));
// 0--> 1 with spill id 1 (attemptNum 0)
dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 0);
handler.handleEvents(Collections.singletonList(dme));
CompositeInputAttemptIdentifier expectedId2 = new CompositeInputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1, 1);
verify(shuffleManager, times(2)).addKnownInput(eq(HOST), eq(PORT), eq(expectedId2), eq(0));
// Let attemptNum 0 be scheduled.
shuffleManager.shuffleInfoEventsMap.get(expectedId2.getInputIdentifier()).scheduledForDownload = true;
// 0--> 1 with spill id 1 (attemptNum 1). This should report exception
dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 1);
handler.handleEvents(Collections.singletonList(dme));
verify(inputContext).killSelf(any(Throwable.class), anyString());
}
use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.
the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents_WithEmptyPartitions.
/**
* In pipelined shuffle, check if processing & exceptions are done correctly when empty
* partitions are sent
*
* @throws IOException
*/
@Test(timeout = 5000)
public void testPipelinedShuffleEvents_WithEmptyPartitions() throws IOException {
InputContext inputContext = createInputContext();
ShuffleManager shuffleManager = createShuffleManager(inputContext);
FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
// 0--> 1 with spill id 0 (attemptNum 0) with empty partitions
BitSet bitSet = new BitSet(4);
bitSet.flip(0, 4);
Event dme = createDataMovementEvent(true, 0, 1, 0, false, bitSet, 4, 0);
handler.handleEvents(Collections.singletonList(dme));
InputAttemptIdentifier expected = new InputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0);
verify(shuffleManager, times(1)).addCompletedInputWithNoData(expected);
// 0--> 1 with spill id 1 (attemptNum 0)
handler.handleEvents(Collections.singletonList(dme));
dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 0);
expected = new InputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1);
verify(shuffleManager, times(2)).addCompletedInputWithNoData(expected);
// Let attemptNum 0 be scheduled.
shuffleManager.shuffleInfoEventsMap.get(expected.getInputIdentifier()).scheduledForDownload = true;
// Now send attemptNum 1. This should throw exception, because attempt #1 is already added
dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 1);
handler.handleEvents(Collections.singletonList(dme));
verify(inputContext).killSelf(any(Throwable.class), anyString());
}
use of org.apache.tez.runtime.library.common.shuffle.FetchedInputAllocator in project tez by apache.
the class TestShuffleInputEventHandlerImpl method testSimple.
@Test(timeout = 5000)
public void testSimple() throws IOException {
InputContext inputContext = mock(InputContext.class);
ShuffleManager shuffleManager = mock(ShuffleManager.class);
FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
int taskIndex = 1;
Event dme = createDataMovementEvent(0, taskIndex, null);
List<Event> eventList = new LinkedList<Event>();
eventList.add(dme);
handler.handleEvents(eventList);
CompositeInputAttemptIdentifier expectedIdentifier = new CompositeInputAttemptIdentifier(taskIndex, 0, PATH_COMPONENT, 1);
verify(shuffleManager).addKnownInput(eq(HOST), eq(PORT), eq(expectedIdentifier), eq(0));
}
Aggregations