Examples with InputContext org.apache.tez.runtime.api.InputContext used on opensource projects

Example 11 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleScheduler method testShutdown.

@Test(timeout = 5000)
public void testShutdown() throws Exception {
    InputContext inputContext = createTezInputContext();
    Configuration conf = new TezConfiguration();
    int numInputs = 10;
    Shuffle shuffle = mock(Shuffle.class);
    MergeManager mergeManager = mock(MergeManager.class);
    final ShuffleSchedulerForTest scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, mergeManager, System.currentTimeMillis(), null, false, 0, "srcName");
    ExecutorService executor = Executors.newFixedThreadPool(1);
    try {
        Future<Void> executorFuture = executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                scheduler.start();
                return null;
            }
        });
        InputAttemptIdentifier[] identifiers = new InputAttemptIdentifier[numInputs];
        for (int i = 0; i < numInputs; i++) {
            CompositeInputAttemptIdentifier inputAttemptIdentifier = new CompositeInputAttemptIdentifier(i, 0, "attempt_", 1);
            scheduler.addKnownMapOutput("host" + i, 10000, 1, inputAttemptIdentifier);
            identifiers[i] = inputAttemptIdentifier;
        }
        MapHost[] mapHosts = new MapHost[numInputs];
        int count = 0;
        for (MapHost mh : scheduler.mapLocations.values()) {
            mapHosts[count++] = mh;
        }
        // Copy succeeded for 1 less host
        for (int i = 0; i < numInputs - 1; i++) {
            MapOutput mapOutput = MapOutput.createMemoryMapOutput(identifiers[i], mock(FetchedInputAllocatorOrderedGrouped.class), 100, false);
            scheduler.copySucceeded(identifiers[i], mapHosts[i], 20, 25, 100, mapOutput, false);
            scheduler.freeHost(mapHosts[i]);
        }
        scheduler.close();
        // Ensure the executor exits, and without an error.
        executorFuture.get();
    } finally {
        scheduler.close();
        executor.shutdownNow();
    }
}

Example 12 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleScheduler method testUseSharedExecutor.

@Test(timeout = 5000)
public void testUseSharedExecutor() throws Exception {
    InputContext inputContext = createTezInputContext();
    Configuration conf = new TezConfiguration();
    int numInputs = 10;
    Shuffle shuffle = mock(Shuffle.class);
    MergeManager mergeManager = mock(MergeManager.class);
    ShuffleSchedulerForTest scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, mergeManager, System.currentTimeMillis(), null, false, 0, "srcName");
    verify(inputContext, times(0)).createTezFrameworkExecutorService(anyInt(), anyString());
    scheduler.close();
    inputContext = createTezInputContext();
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_FETCHER_USE_SHARED_POOL, true);
    scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, mergeManager, System.currentTimeMillis(), null, false, 0, "srcName");
    verify(inputContext).createTezFrameworkExecutorService(anyInt(), anyString());
    scheduler.close();
}

Example 13 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleScheduler method testNumParallelScheduledFetchers.

@Test(timeout = 10000)
public void testNumParallelScheduledFetchers() throws IOException, InterruptedException {
    InputContext inputContext = createTezInputContext();
    Configuration conf = new TezConfiguration();
    // Allow 10 parallel copies at once.
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_PARALLEL_COPIES, 10);
    int numInputs = 50;
    Shuffle shuffle = mock(Shuffle.class);
    MergeManager mergeManager = mock(MergeManager.class);
    final ShuffleSchedulerForTest scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, mergeManager, System.currentTimeMillis(), null, false, 0, "srcName", true);
    Future<Void> executorFuture = null;
    ExecutorService executor = Executors.newFixedThreadPool(1);
    try {
        executorFuture = executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                scheduler.start();
                return null;
            }
        });
        InputAttemptIdentifier[] identifiers = new InputAttemptIdentifier[numInputs];
        // Schedule all copies.
        for (int i = 0; i < numInputs; i++) {
            CompositeInputAttemptIdentifier inputAttemptIdentifier = new CompositeInputAttemptIdentifier(i, 0, "attempt_", 1);
            scheduler.addKnownMapOutput("host" + i, 10000, 1, inputAttemptIdentifier);
            identifiers[i] = inputAttemptIdentifier;
        }
        // Sleep for a bit to allow the copies to be scheduled.
        Thread.sleep(2000l);
        assertEquals(10, scheduler.numFetchersCreated.get());
    } finally {
        scheduler.close();
        if (executorFuture != null) {
            executorFuture.cancel(true);
        }
        executor.shutdownNow();
    }
}

Example 14 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents_WithOutOfOrderAttempts.

/**
 * In pipelined shuffle, check if processing & exceptions are done correctly when attempts are
 * received in out of order fashion (e.g attemptNum 1 arrives before attemptNum 0)
 *
 * @throws IOException
 */
@Test(timeout = 5000)
public void testPipelinedShuffleEvents_WithOutOfOrderAttempts() throws IOException {
    InputContext inputContext = createInputContext();
    ShuffleManager shuffleManager = createShuffleManager(inputContext);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    // 0--> 1 with spill id 0 (attemptNum 1).  attemptNum 0 is not sent.
    Event dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 1);
    handler.handleEvents(Collections.singletonList(dme));
    CompositeInputAttemptIdentifier expected = new CompositeInputAttemptIdentifier(1, 1, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1, 1);
    verify(shuffleManager, times(1)).addKnownInput(eq(HOST), eq(PORT), eq(expected), eq(0));
    // Let attemptNum 1 be scheduled.
    shuffleManager.shuffleInfoEventsMap.get(expected.getInputIdentifier()).scheduledForDownload = true;
    // Now send attemptNum 0.  This should throw exception, because attempt #1 is already added
    dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    verify(inputContext).killSelf(any(Throwable.class), anyString());
}

Example 15 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testPipelinedShuffleEvents.

/**
 * In pipelined shuffle, check if multiple attempt numbers are processed and
 * exceptions are reported properly.
 *
 * @throws IOException
 */
@Test(timeout = 5000)
public void testPipelinedShuffleEvents() throws IOException {
    InputContext inputContext = createInputContext();
    ShuffleManager shuffleManager = createShuffleManager(inputContext);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    // 0--> 1 with spill id 0 (attemptNum 0)
    Event dme = createDataMovementEvent(true, 0, 1, 0, false, new BitSet(), 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    CompositeInputAttemptIdentifier expectedId1 = new CompositeInputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 0, 1);
    verify(shuffleManager, times(1)).addKnownInput(eq(HOST), eq(PORT), eq(expectedId1), eq(0));
    // 0--> 1 with spill id 1 (attemptNum 0)
    dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 0);
    handler.handleEvents(Collections.singletonList(dme));
    CompositeInputAttemptIdentifier expectedId2 = new CompositeInputAttemptIdentifier(1, 0, PATH_COMPONENT, false, InputAttemptIdentifier.SPILL_INFO.INCREMENTAL_UPDATE, 1, 1);
    verify(shuffleManager, times(2)).addKnownInput(eq(HOST), eq(PORT), eq(expectedId2), eq(0));
    // Let attemptNum 0 be scheduled.
    shuffleManager.shuffleInfoEventsMap.get(expectedId2.getInputIdentifier()).scheduledForDownload = true;
    // 0--> 1 with spill id 1 (attemptNum 1).  This should report exception
    dme = createDataMovementEvent(true, 0, 1, 1, false, new BitSet(), 4, 1);
    handler.handleEvents(Collections.singletonList(dme));
    verify(inputContext).killSelf(any(Throwable.class), anyString());
}