Examples with InputContext org.apache.tez.runtime.api.InputContext used on opensource projects

Example 51 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleScheduler method createScheduler.

private ShuffleSchedulerForTest createScheduler(long startTime, int numInputs, Shuffle shuffle, Configuration conf) throws IOException {
    InputContext inputContext = createTezInputContext();
    MergeManager mergeManager = mock(MergeManager.class);
    final ShuffleSchedulerForTest scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, mergeManager, startTime, null, false, 0, "srcName");
    return scheduler;
}

Example 52 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleScheduler method testSimpleFlow.

@Test(timeout = 5000)
public void testSimpleFlow() throws Exception {
    InputContext inputContext = createTezInputContext();
    Configuration conf = new TezConfiguration();
    int numInputs = 10;
    Shuffle shuffle = mock(Shuffle.class);
    MergeManager mergeManager = mock(MergeManager.class);
    final ShuffleSchedulerForTest scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, mergeManager, System.currentTimeMillis(), null, false, 0, "srcName");
    ExecutorService executor = Executors.newFixedThreadPool(1);
    try {
        Future<Void> executorFuture = executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                scheduler.start();
                return null;
            }
        });
        InputAttemptIdentifier[] identifiers = new InputAttemptIdentifier[numInputs];
        for (int i = 0; i < numInputs; i++) {
            CompositeInputAttemptIdentifier inputAttemptIdentifier = new CompositeInputAttemptIdentifier(i, 0, "attempt_", 1);
            scheduler.addKnownMapOutput("host" + i, 10000, 1, inputAttemptIdentifier);
            identifiers[i] = inputAttemptIdentifier;
        }
        MapHost[] mapHosts = new MapHost[numInputs];
        int count = 0;
        for (MapHost mh : scheduler.mapLocations.values()) {
            mapHosts[count++] = mh;
        }
        for (int i = 0; i < numInputs; i++) {
            MapOutput mapOutput = MapOutput.createMemoryMapOutput(identifiers[i], mock(FetchedInputAllocatorOrderedGrouped.class), 100, false);
            scheduler.copySucceeded(identifiers[i], mapHosts[i], 20, 25, 100, mapOutput, false);
            scheduler.freeHost(mapHosts[i]);
        }
        verify(inputContext, atLeast(numInputs)).notifyProgress();
        // Ensure the executor exits, and without an error.
        executorFuture.get();
    } finally {
        scheduler.close();
        executor.shutdownNow();
    }
}

Example 53 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleScheduler method testShutdownWithInterrupt.

@Test(timeout = 30000)
public void testShutdownWithInterrupt() throws Exception {
    InputContext inputContext = createTezInputContext();
    Configuration conf = new TezConfiguration();
    int numInputs = 10;
    Shuffle shuffle = mock(Shuffle.class);
    MergeManager mergeManager = mock(MergeManager.class);
    final ShuffleSchedulerForTest scheduler = new ShuffleSchedulerForTest(inputContext, conf, numInputs, shuffle, mergeManager, mergeManager, System.currentTimeMillis(), null, false, 0, "srcName");
    ExecutorService executor = Executors.newFixedThreadPool(1);
    Future<Void> executorFuture = executor.submit(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            scheduler.start();
            return null;
        }
    });
    InputAttemptIdentifier[] identifiers = new InputAttemptIdentifier[numInputs];
    for (int i = 0; i < numInputs; i++) {
        CompositeInputAttemptIdentifier inputAttemptIdentifier = new CompositeInputAttemptIdentifier(i, 0, "attempt_", 1);
        scheduler.addKnownMapOutput("host" + i, 10000, 1, inputAttemptIdentifier);
        identifiers[i] = inputAttemptIdentifier;
    }
    MapHost[] mapHosts = new MapHost[numInputs];
    int count = 0;
    for (MapHost mh : scheduler.mapLocations.values()) {
        mapHosts[count++] = mh;
    }
    // Copy succeeded for 1 less host
    for (int i = 0; i < numInputs - 1; i++) {
        MapOutput mapOutput = MapOutput.createMemoryMapOutput(identifiers[i], mock(FetchedInputAllocatorOrderedGrouped.class), 100, false);
        scheduler.copySucceeded(identifiers[i], mapHosts[i], 20, 25, 100, mapOutput, false);
        scheduler.freeHost(mapHosts[i]);
    }
    try {
        // Close the scheduler on different thread to trigger interrupt
        Thread thread = new Thread(new Runnable() {

            @Override
            public void run() {
                scheduler.close();
            }
        });
        thread.start();
        thread.join();
    } finally {
        assertTrue("Fetcher executor should be shutdown, but still running", scheduler.hasFetcherExecutorStopped());
        executor.shutdownNow();
    }
}

Example 54 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleInputEventHandlerImpl method testCurrentPartitionEmpty.

@Test(timeout = 5000)
public void testCurrentPartitionEmpty() throws IOException {
    InputContext inputContext = mock(InputContext.class);
    ShuffleManager shuffleManager = mock(ShuffleManager.class);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    int taskIndex = 1;
    Event dme = createDataMovementEvent(0, taskIndex, createEmptyPartitionByteString(0));
    List<Event> eventList = new LinkedList<Event>();
    eventList.add(dme);
    handler.handleEvents(eventList);
    InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(taskIndex, 0);
    verify(shuffleManager).addCompletedInputWithNoData(eq(expectedIdentifier));
}

Example 55 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestMergeManager method createMockInputContext.

private InputContext createMockInputContext(String uniqueId, long mem) {
    InputContext inputContext = mock(InputContext.class);
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn(mem).when(inputContext).getTotalMemoryAvailableToTask();
    doReturn("srcVertexName").when(inputContext).getSourceVertexName();
    doReturn(uniqueId).when(inputContext).getUniqueIdentifier();
    return inputContext;
}