Examples with InputContext org.apache.tez.runtime.api.InputContext used on opensource projects

Example 41 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestMemoryDistributor method testScalingNoProcessor.

@Test(timeout = 5000)
public void testScalingNoProcessor() throws TezException {
    MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
    dist.setJvmMemory(10000l);
    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
    dist.makeInitialAllocations();
    // Total available: 70% of 10K = 7000
    // 3 requests - 10K, 10K, 5K
    // Scale down to - 2800, 2800, 1400
    assertEquals(2800, e1Callback.assigned);
    assertEquals(2800, e2Callback.assigned);
    assertEquals(1400, e3Callback.assigned);
}

Example 42 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestLogicalIOProcessorRuntimeTask method cleanupAndTest.

private void cleanupAndTest(LogicalIOProcessorRuntimeTask lio) throws InterruptedException {
    ProcessorContext procContext = lio.getProcessorContext();
    List<InputContext> inputContexts = new LinkedList<InputContext>();
    inputContexts.addAll(lio.getInputContexts());
    List<OutputContext> outputContexts = new LinkedList<OutputContext>();
    outputContexts.addAll(lio.getOutputContexts());
    lio.cleanup();
    assertTrue(procContext.getUserPayload() == null);
    assertTrue(procContext.getObjectRegistry() == null);
    for (InputContext inputContext : inputContexts) {
        assertTrue(inputContext.getUserPayload() == null);
        assertTrue(inputContext.getObjectRegistry() == null);
    }
    for (OutputContext outputContext : outputContexts) {
        assertTrue(outputContext.getUserPayload() == null);
        assertTrue(outputContext.getObjectRegistry() == null);
    }
    boolean localMode = lio.tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
    if (localMode) {
        assertEquals(1, lio.inputSpecs.size());
        assertEquals(1, lio.outputSpecs.size());
        assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
    } else {
        assertEquals(0, lio.inputSpecs.size());
        assertEquals(0, lio.outputSpecs.size());
        assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
    }
    assertEquals(0, lio.inputsMap.size());
    assertEquals(0, lio.inputContextMap.size());
    assertEquals(0, lio.outputsMap.size());
    assertEquals(0, lio.outputContextMap.size());
    assertNull(lio.groupInputsMap);
    assertNull(lio.processor);
    assertNull(lio.processorContext);
    assertEquals(0, lio.runInputMap.size());
    assertEquals(0, lio.runOutputMap.size());
    assertEquals(0, lio.eventsToBeProcessed.size());
    assertNull(lio.eventRouterThread);
}

Example 43 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestValuesIterator method createTezInputContext.

private InputContext createTezInputContext() {
    TezCounters counters = new TezCounters();
    InputContext inputContext = mock(InputContext.class);
    doReturn(1024 * 1024 * 100l).when(inputContext).getTotalMemoryAvailableToTask();
    doReturn(counters).when(inputContext).getCounters();
    doReturn(1).when(inputContext).getInputIndex();
    doReturn("srcVertex").when(inputContext).getSourceVertexName();
    doReturn(1).when(inputContext).getTaskVertexIndex();
    doReturn(UserPayload.create(ByteBuffer.wrap(new byte[1024]))).when(inputContext).getUserPayload();
    return inputContext;
}

Example 44 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestShuffleManager method testMultiplePartitions.

/**
 * One reducer fetches multiple partitions from each mapper.
 * For a given mapper, the reducer sends DataMovementEvents for several
 * partitions, wait for some time and then send DataMovementEvents for the
 * rest of the partitions. Then do the same thing for the next mapper.
 * Verify ShuffleManager is able to get all the events.
 */
@Test(timeout = 50000)
public void testMultiplePartitions() throws Exception {
    final int numOfMappers = 3;
    final int numOfPartitions = 5;
    final int firstPart = 2;
    InputContext inputContext = createInputContext();
    ShuffleManagerForTest shuffleManager = createShuffleManager(inputContext, numOfMappers * numOfPartitions);
    FetchedInputAllocator inputAllocator = mock(FetchedInputAllocator.class);
    ShuffleInputEventHandlerImpl handler = new ShuffleInputEventHandlerImpl(inputContext, shuffleManager, inputAllocator, null, false, 0, false);
    shuffleManager.run();
    List<Event> eventList = new LinkedList<Event>();
    // The physical input index within the reduce task
    int targetIndex = 0;
    for (int i = 0; i < numOfMappers; i++) {
        String mapperHost = "host" + i;
        // The physical output index within the map task
        int srcIndex = 20;
        // Send the first batch of DataMovementEvents
        eventList.clear();
        for (int j = 0; j < firstPart; j++) {
            Event dme = createDataMovementEvent(mapperHost, srcIndex++, targetIndex++);
            eventList.add(dme);
        }
        handler.handleEvents(eventList);
        Thread.sleep(500);
        // Send the second batch of DataMovementEvents
        eventList.clear();
        for (int j = 0; j < numOfPartitions - firstPart; j++) {
            Event dme = createDataMovementEvent(mapperHost, srcIndex++, targetIndex++);
            eventList.add(dme);
        }
        handler.handleEvents(eventList);
    }
    int waitCount = 100;
    while (waitCount-- > 0 && !(shuffleManager.isFetcherExecutorShutdown() && numOfMappers * numOfPartitions == shuffleManager.getNumOfCompletedInputs())) {
        Thread.sleep(100);
    }
    assertTrue(shuffleManager.isFetcherExecutorShutdown());
    assertEquals(numOfMappers * numOfPartitions, shuffleManager.getNumOfCompletedInputs());
}

Example 45 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestFetcher method testWithRetry.

@Test(timeout = 5000)
@SuppressWarnings("unchecked")
public void testWithRetry() throws Exception {
    Configuration conf = new TezConfiguration();
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_READ_TIMEOUT, 3000);
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_SHUFFLE_CONNECT_TIMEOUT, 3000);
    ShuffleScheduler scheduler = mock(ShuffleScheduler.class);
    MergeManager merger = mock(MergeManager.class);
    Shuffle shuffle = mock(Shuffle.class);
    InputContext inputContext = mock(InputContext.class);
    when(inputContext.getCounters()).thenReturn(new TezCounters());
    when(inputContext.getSourceVertexName()).thenReturn("");
    when(inputContext.getApplicationId()).thenReturn(ApplicationId.newInstance(0, 1));
    HttpConnectionParams httpConnectionParams = ShuffleUtils.getHttpConnectionParams(conf);
    final MapHost host = new MapHost(HOST, PORT, 1, 1);
    FetcherOrderedGrouped mockFetcher = new FetcherOrderedGrouped(null, scheduler, merger, shuffle, null, false, 0, null, conf, false, HOST, PORT, "src vertex", host, ioErrsCounter, wrongLengthErrsCounter, badIdErrsCounter, wrongMapErrsCounter, connectionErrsCounter, wrongReduceErrsCounter, APP_ID, DAG_ID, false, false, true, false);
    final FetcherOrderedGrouped fetcher = spy(mockFetcher);
    final List<InputAttemptIdentifier> srcAttempts = Arrays.asList(new InputAttemptIdentifier(0, 1, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_0"), new InputAttemptIdentifier(1, 2, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_1"), new InputAttemptIdentifier(3, 4, InputAttemptIdentifier.PATH_PREFIX + "pathComponent_3"));
    doReturn(srcAttempts).when(scheduler).getMapsForHost(host);
    doReturn(true).when(fetcher).setupConnection(any(MapHost.class), any(Collection.class));
    URL url = ShuffleUtils.constructInputURL("http://" + HOST + ":" + PORT + "/mapOutput?job=job_123&&reduce=1&map=", srcAttempts, false);
    fetcher.httpConnection = new FakeHttpConnection(url, null, "", null);
    doAnswer(new Answer<MapOutput>() {

        @Override
        public MapOutput answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            MapOutput mapOutput = mock(MapOutput.class);
            doReturn(MapOutput.Type.MEMORY).when(mapOutput).getType();
            doReturn(args[0]).when(mapOutput).getAttemptIdentifier();
            return mapOutput;
        }
    }).when(merger).reserve(any(InputAttemptIdentifier.class), anyInt(), anyInt(), anyInt());
    // Create read timeout when reading data
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            // Emulate host down for 4 seconds.
            Thread.sleep(4000);
            doReturn(false).when(fetcher).setupConnection(any(MapHost.class), any(Collection.class));
            // Throw IOException when fetcher tries to connect again to the same node
            throw new FetcherReadTimeoutException("creating fetcher socket read timeout exception");
        }
    }).when(fetcher).copyMapOutput(any(MapHost.class), any(DataInputStream.class), any(InputAttemptIdentifier.class));
    try {
        fetcher.copyFromHost(host);
    } catch (IOException e) {
    // ignore
    }
    // setup connection should be called twice (1 for connect and another for retry)
    verify(fetcher, times(2)).setupConnection(any(MapHost.class), any(Collection.class));
    // since copyMapOutput consistently fails, it should call copyFailed once
    verify(scheduler, times(1)).copyFailed(any(InputAttemptIdentifier.class), any(MapHost.class), anyBoolean(), anyBoolean(), anyBoolean());
    verify(fetcher, times(1)).putBackRemainingMapOutputs(any(MapHost.class));
    verify(scheduler, times(3)).putBackKnownMapOutput(any(MapHost.class), any(InputAttemptIdentifier.class));
    // Verify by stopping the fetcher abruptly
    try {
        // flag to indicate fetcher stopped
        fetcher.stopped = false;
        fetcher.copyFromHost(host);
        verify(fetcher, times(2)).putBackRemainingMapOutputs(any(MapHost.class));
    } catch (IOException e) {
    // ignore
    }
}