Examples with TezCounters org.apache.tez.common.counters.TezCounters used on opensource projects

Example 36 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestShuffleInputEventHandlerImpl method createInputContext.

private InputContext createInputContext() throws IOException {
    DataOutputBuffer port_dob = new DataOutputBuffer();
    port_dob.writeInt(PORT);
    final ByteBuffer shuffleMetaData = ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());
    port_dob.close();
    ExecutionContext executionContext = mock(ExecutionContext.class);
    doReturn(HOST).when(executionContext).getHostName();
    InputContext inputContext = mock(InputContext.class);
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn("sourceVertex").when(inputContext).getSourceVertexName();
    doReturn("taskVertex").when(inputContext).getTaskVertexName();
    doReturn(shuffleMetaData).when(inputContext).getServiceProviderMetaData(conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT));
    doReturn(executionContext).when(inputContext).getExecutionContext();
    when(inputContext.createTezFrameworkExecutorService(anyInt(), anyString())).thenAnswer(new Answer<ExecutorService>() {

        @Override
        public ExecutorService answer(InvocationOnMock invocation) throws Throwable {
            return sharedExecutor.createExecutorService(invocation.getArgumentAt(0, Integer.class), invocation.getArgumentAt(1, String.class));
        }
    });
    return inputContext;
}

Example 37 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestShuffleManager method createInputContext.

private InputContext createInputContext() throws IOException {
    DataOutputBuffer port_dob = new DataOutputBuffer();
    port_dob.writeInt(PORT);
    final ByteBuffer shuffleMetaData = ByteBuffer.wrap(port_dob.getData(), 0, port_dob.getLength());
    port_dob.close();
    ExecutionContext executionContext = mock(ExecutionContext.class);
    doReturn(FETCHER_HOST).when(executionContext).getHostName();
    InputContext inputContext = mock(InputContext.class);
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn("sourceVertex").when(inputContext).getSourceVertexName();
    doReturn("taskVertex").when(inputContext).getTaskVertexName();
    doReturn(shuffleMetaData).when(inputContext).getServiceProviderMetaData(conf.get(TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID, TezConfiguration.TEZ_AM_SHUFFLE_AUXILIARY_SERVICE_ID_DEFAULT));
    doReturn(executionContext).when(inputContext).getExecutionContext();
    when(inputContext.createTezFrameworkExecutorService(anyInt(), anyString())).thenAnswer(new Answer<ExecutorService>() {

        @Override
        public ExecutorService answer(InvocationOnMock invocation) throws Throwable {
            return sharedExecutor.createExecutorService(invocation.getArgumentAt(0, Integer.class), invocation.getArgumentAt(1, String.class));
        }
    });
    return inputContext;
}

Example 38 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestFetcher method createMockInputContext.

private InputContext createMockInputContext() {
    InputContext inputContext = mock(InputContext.class);
    doReturn(APP_ID).when(inputContext).getApplicationId();
    doReturn(DAG_ID).when(inputContext).getDagIdentifier();
    doReturn(new TezCounters()).when(inputContext).getCounters();
    doReturn("src vertex").when(inputContext).getSourceVertexName();
    doReturn("task_Vertex").when(inputContext).getTaskVertexName();
    return inputContext;
}

Example 39 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestOrderedGroupedKVInput method createMockInputContext.

private InputContext createMockInputContext() throws IOException {
    InputContext inputContext = mock(InputContext.class);
    Configuration conf = new TezConfiguration();
    UserPayload payLoad = TezUtils.createUserPayloadFromConf(conf);
    String[] workingDirs = new String[] { "workDir1" };
    TezCounters counters = new TezCounters();
    doReturn(payLoad).when(inputContext).getUserPayload();
    doReturn(workingDirs).when(inputContext).getWorkDirs();
    doReturn(200 * 1024 * 1024l).when(inputContext).getTotalMemoryAvailableToTask();
    doReturn(counters).when(inputContext).getCounters();
    doReturn(new Configuration(false)).when(inputContext).getContainerConfiguration();
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args[1] instanceof MemoryUpdateCallbackHandler) {
                MemoryUpdateCallbackHandler memUpdateCallbackHandler = (MemoryUpdateCallbackHandler) args[1];
                memUpdateCallbackHandler.memoryAssigned(200 * 1024 * 1024);
            } else {
                Assert.fail();
            }
            return null;
        }
    }).when(inputContext).requestInitialMemory(any(long.class), any(MemoryUpdateCallbackHandler.class));
    return inputContext;
}

Example 40 with TezCounters

use of org.apache.tez.common.counters.TezCounters in project tez by apache.

the class TestOrderedGroupedKVInput method testMergeConfig.

@Test
public void testMergeConfig() throws IOException, TezException {
    Configuration baseConf = new Configuration(false);
    baseConf.set("base-key", "base-value");
    Configuration payloadConf = new Configuration(false);
    payloadConf.set("local-key", "local-value");
    InputContext inputContext = mock(InputContext.class);
    UserPayload payLoad = TezUtils.createUserPayloadFromConf(payloadConf);
    String[] workingDirs = new String[] { "workDir1" };
    TezCounters counters = new TezCounters();
    doReturn(payLoad).when(inputContext).getUserPayload();
    doReturn(workingDirs).when(inputContext).getWorkDirs();
    doReturn(counters).when(inputContext).getCounters();
    doReturn(baseConf).when(inputContext).getContainerConfiguration();
    OrderedGroupedKVInput input = new OrderedGroupedKVInput(inputContext, 1);
    input.initialize();
    Configuration mergedConf = input.conf;
    assertEquals("base-value", mergedConf.get("base-key"));
    assertEquals("local-value", mergedConf.get("local-key"));
}