Examples with OutputContext org.apache.tez.runtime.api.OutputContext used on opensource projects

Example 31 with OutputContext

use of org.apache.tez.runtime.api.OutputContext in project tez by apache.

the class TestMemoryDistributor method createTestOutputContext.

protected OutputContext createTestOutputContext() {
    OutputContext context = mock(OutputContext.class);
    doReturn("output").when(context).getDestinationVertexName();
    doReturn("task").when(context).getTaskVertexName();
    return context;
}

Example 32 with OutputContext

use of org.apache.tez.runtime.api.OutputContext in project tez by apache.

the class TestMemoryDistributor method testScalingNoProcessor.

@Test(timeout = 5000)
public void testScalingNoProcessor() throws TezException {
    MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
    dist.setJvmMemory(10000l);
    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
    dist.makeInitialAllocations();
    // Total available: 70% of 10K = 7000
    // 3 requests - 10K, 10K, 5K
    // Scale down to - 2800, 2800, 1400
    assertEquals(2800, e1Callback.assigned);
    assertEquals(2800, e2Callback.assigned);
    assertEquals(1400, e3Callback.assigned);
}

Example 33 with OutputContext

use of org.apache.tez.runtime.api.OutputContext in project tez by apache.

the class TestLogicalIOProcessorRuntimeTask method cleanupAndTest.

private void cleanupAndTest(LogicalIOProcessorRuntimeTask lio) throws InterruptedException {
    ProcessorContext procContext = lio.getProcessorContext();
    List<InputContext> inputContexts = new LinkedList<InputContext>();
    inputContexts.addAll(lio.getInputContexts());
    List<OutputContext> outputContexts = new LinkedList<OutputContext>();
    outputContexts.addAll(lio.getOutputContexts());
    lio.cleanup();
    assertTrue(procContext.getUserPayload() == null);
    assertTrue(procContext.getObjectRegistry() == null);
    for (InputContext inputContext : inputContexts) {
        assertTrue(inputContext.getUserPayload() == null);
        assertTrue(inputContext.getObjectRegistry() == null);
    }
    for (OutputContext outputContext : outputContexts) {
        assertTrue(outputContext.getUserPayload() == null);
        assertTrue(outputContext.getObjectRegistry() == null);
    }
    boolean localMode = lio.tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
    if (localMode) {
        assertEquals(1, lio.inputSpecs.size());
        assertEquals(1, lio.outputSpecs.size());
        assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
    } else {
        assertEquals(0, lio.inputSpecs.size());
        assertEquals(0, lio.outputSpecs.size());
        assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
    }
    assertEquals(0, lio.inputsMap.size());
    assertEquals(0, lio.inputContextMap.size());
    assertEquals(0, lio.outputsMap.size());
    assertEquals(0, lio.outputContextMap.size());
    assertNull(lio.groupInputsMap);
    assertNull(lio.processor);
    assertNull(lio.processorContext);
    assertEquals(0, lio.runInputMap.size());
    assertEquals(0, lio.runOutputMap.size());
    assertEquals(0, lio.eventsToBeProcessed.size());
    assertNull(lio.eventRouterThread);
}

Example 34 with OutputContext

use of org.apache.tez.runtime.api.OutputContext in project tez by apache.

the class TestPipelinedSorter method createMockOutputContext.

private static OutputContext createMockOutputContext(TezCounters counters, ApplicationId appId, String uniqueId, String auxiliaryService) throws IOException {
    OutputContext outputContext = mock(OutputContext.class);
    ExecutionContext execContext = new ExecutionContextImpl("localhost");
    DataOutputBuffer serviceProviderMetaData = new DataOutputBuffer();
    serviceProviderMetaData.writeInt(80);
    doReturn(ByteBuffer.wrap(serviceProviderMetaData.getData())).when(outputContext).getServiceProviderMetaData(auxiliaryService);
    doReturn(execContext).when(outputContext).getExecutionContext();
    doReturn(mock(OutputStatisticsReporter.class)).when(outputContext).getStatisticsReporter();
    doReturn(counters).when(outputContext).getCounters();
    doReturn(appId).when(outputContext).getApplicationId();
    doReturn(1).when(outputContext).getDAGAttemptNumber();
    doReturn("dagName").when(outputContext).getDAGName();
    doReturn("destinationVertexName").when(outputContext).getDestinationVertexName();
    doReturn(1).when(outputContext).getOutputIndex();
    doReturn(1).when(outputContext).getTaskAttemptNumber();
    doReturn(1).when(outputContext).getTaskIndex();
    doReturn(1).when(outputContext).getTaskVertexIndex();
    doReturn("vertexName").when(outputContext).getTaskVertexName();
    doReturn(uniqueId).when(outputContext).getUniqueIdentifier();
    Path outDirBase = new Path(workDir, "outDir_" + uniqueId);
    String[] outDirs = new String[] { outDirBase.toString() };
    doReturn(outDirs).when(outputContext).getWorkDirs();
    return outputContext;
}

Example 35 with OutputContext

use of org.apache.tez.runtime.api.OutputContext in project tez by apache.

the class TestDefaultSorter method testWithMultipleSpillsWithFinalMergeDisabled.

@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testWithMultipleSpillsWithFinalMergeDisabled() throws IOException {
    OutputContext context = createTezOutputContext();
    conf.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_ENABLE_FINAL_MERGE_IN_OUTPUT, false);
    conf.setLong(TezRuntimeConfiguration.TEZ_RUNTIME_IO_SORT_MB, 4);
    conf.setInt(TezRuntimeConfiguration.TEZ_RUNTIME_INDEX_CACHE_MEMORY_LIMIT_BYTES, 1);
    MemoryUpdateCallbackHandler handler = new MemoryUpdateCallbackHandler();
    context.requestInitialMemory(ExternalSorter.getInitialMemoryRequirement(conf, context.getTotalMemoryAvailableToTask()), handler);
    DefaultSorter sorter = new DefaultSorter(context, conf, 1, handler.getMemoryAssigned());
    writeData(sorter, 10000, 1000);
    int spillCount = sorter.getNumSpills();
    ArgumentCaptor<List> eventCaptor = ArgumentCaptor.forClass(List.class);
    verify(context, times(1)).sendEvents(eventCaptor.capture());
    List<Event> events = eventCaptor.getValue();
    int spillIndex = 0;
    for (Event event : events) {
        if (event instanceof CompositeDataMovementEvent) {
            CompositeDataMovementEvent cdme = (CompositeDataMovementEvent) event;
            ShuffleUserPayloads.DataMovementEventPayloadProto shufflePayload = ShuffleUserPayloads.DataMovementEventPayloadProto.parseFrom(ByteString.copyFrom(cdme.getUserPayload()));
            assertTrue(shufflePayload.getPathComponent().equalsIgnoreCase(UniqueID + "_" + spillIndex));
            verifyOutputPermissions(shufflePayload.getPathComponent());
            spillIndex++;
        }
    }
    assertTrue(spillIndex == spillCount);
    verifyCounters(sorter, context);
}