Examples with InputContext org.apache.tez.runtime.api.InputContext used on opensource projects

Example 36 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestMemoryDistributor method testScalingProcessor.

@Test(timeout = 5000)
public void testScalingProcessor() throws TezException {
    MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
    dist.setJvmMemory(10000l);
    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc1);
    // Fourth request - processor
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    ProcessorContext e4ProcessorContext1 = createTestProcessortContext();
    ProcessorDescriptor e4ProcessorDesc1 = createTestProcessorDescriptor();
    dist.requestMemory(5000, e4Callback, e4ProcessorContext1, e4ProcessorDesc1);
    dist.makeInitialAllocations();
    // Total available: 70% of 10K = 7000
    // 4 requests - 10K, 10K, 5K, 5K
    // Scale down to - 2333.33, 2333.33, 1166.66, 1166.66
    assertTrue(e1Callback.assigned >= 2333 && e1Callback.assigned <= 2334);
    assertTrue(e2Callback.assigned >= 2333 && e2Callback.assigned <= 2334);
    assertTrue(e3Callback.assigned >= 1166 && e3Callback.assigned <= 1167);
    assertTrue(e4Callback.assigned >= 1166 && e4Callback.assigned <= 1167);
}

Example 37 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestMemoryDistributor method testScalingDisabled.

@Test(timeout = 5000)
public void testScalingDisabled() throws TezException {
    // Real world values
    Configuration conf = new Configuration(this.conf);
    conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_ENABLED, false);
    MemoryDistributor dist = new MemoryDistributor(2, 0, conf);
    dist.setJvmMemory(207093760l);
    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1);
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(144965632l, e2Callback, e2InputContext2, e2InDesc2);
    dist.makeInitialAllocations();
    assertEquals(104857600l, e1Callback.assigned);
    assertEquals(144965632l, e2Callback.assigned);
}

Example 38 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestMemoryDistributor method testScalingNoProcessor2.

@Test(timeout = 5000)
public void testScalingNoProcessor2() throws TezException {
    // Real world values
    MemoryDistributor dist = new MemoryDistributor(2, 0, conf);
    dist.setJvmMemory(209715200l);
    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(104857600l, e1Callback, e1InputContext1, e1InDesc1);
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(157286400l, e2Callback, e2InputContext2, e2InDesc2);
    dist.makeInitialAllocations();
    assertEquals(58720256l, e1Callback.assigned);
    assertEquals(88080384l, e2Callback.assigned);
}

Example 39 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestMemoryDistributor method testReserveFractionConfigured.

@Test(timeout = 5000)
public void testReserveFractionConfigured() throws TezException {
    Configuration conf = new Configuration(this.conf);
    conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.5d);
    MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
    dist.setJvmMemory(10000l);
    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc2 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc2);
    dist.makeInitialAllocations();
    // Total available: 50% of 10K = 7000
    // 3 requests - 10K, 10K, 5K
    // Scale down to - 2000, 2000, 1000
    assertEquals(2000, e1Callback.assigned);
    assertEquals(2000, e2Callback.assigned);
    assertEquals(1000, e3Callback.assigned);
}

Example 40 with InputContext

use of org.apache.tez.runtime.api.InputContext in project tez by apache.

the class TestMemoryDistributor method createTestInputContext.

protected InputContext createTestInputContext() {
    InputContext context = mock(InputContext.class);
    doReturn("input").when(context).getSourceVertexName();
    doReturn("task").when(context).getTaskVertexName();
    return context;
}