Examples with ProcessorContext org.apache.tez.runtime.api.ProcessorContext used on opensource projects

Example 1 with ProcessorContext

use of org.apache.tez.runtime.api.ProcessorContext in project tez by apache.

the class TestMemoryDistributor method createTestProcessortContext.

protected ProcessorContext createTestProcessortContext() {
    ProcessorContext context = mock(ProcessorContext.class);
    doReturn("task").when(context).getTaskVertexName();
    return context;
}

Example 2 with ProcessorContext

use of org.apache.tez.runtime.api.ProcessorContext in project tez by apache.

the class TestMemoryDistributor method testScalingProcessor.

@Test(timeout = 5000)
public void testScalingProcessor() throws TezException {
    MemoryDistributor dist = new MemoryDistributor(2, 1, conf);
    dist.setJvmMemory(10000l);
    // First request
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor();
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
    // Second request
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor();
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
    // Third request - output
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(5000, e3Callback, e3OutputContext1, e3OutDesc1);
    // Fourth request - processor
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    ProcessorContext e4ProcessorContext1 = createTestProcessortContext();
    ProcessorDescriptor e4ProcessorDesc1 = createTestProcessorDescriptor();
    dist.requestMemory(5000, e4Callback, e4ProcessorContext1, e4ProcessorDesc1);
    dist.makeInitialAllocations();
    // Total available: 70% of 10K = 7000
    // 4 requests - 10K, 10K, 5K, 5K
    // Scale down to - 2333.33, 2333.33, 1166.66, 1166.66
    assertTrue(e1Callback.assigned >= 2333 && e1Callback.assigned <= 2334);
    assertTrue(e2Callback.assigned >= 2333 && e2Callback.assigned <= 2334);
    assertTrue(e3Callback.assigned >= 1166 && e3Callback.assigned <= 1167);
    assertTrue(e4Callback.assigned >= 1166 && e4Callback.assigned <= 1167);
}

Example 3 with ProcessorContext

use of org.apache.tez.runtime.api.ProcessorContext in project hive by apache.

the class TestTezProcessor method testCurrentUserCredentialsAreMergedOnInitialize.

@Test
public void testCurrentUserCredentialsAreMergedOnInitialize() throws IOException {
    ProcessorContext processorContext = mockProcessorContext();
    TezProcessor tezProcessor = new TezProcessor(processorContext);
    Text tokenId = new Text("TOKEN_KIND");
    Token<TokenIdentifier> token = new Token<>("hello".getBytes(), null, new Text("TEST_TOKEN_KIND"), new Text("TEST_TOKEN_SERVICE"));
    UserGroupInformation.getCurrentUser().addToken(tokenId, token);
    tezProcessor.initialize();
    JobConf conf = tezProcessor.getConf();
    Assert.assertEquals(token, conf.getCredentials().getToken(tokenId));
}

Example 4 with ProcessorContext

use of org.apache.tez.runtime.api.ProcessorContext in project tez by apache.

the class TestLogicalIOProcessorRuntimeTask method cleanupAndTest.

private void cleanupAndTest(LogicalIOProcessorRuntimeTask lio) throws InterruptedException {
    ProcessorContext procContext = lio.getProcessorContext();
    List<InputContext> inputContexts = new LinkedList<InputContext>();
    inputContexts.addAll(lio.getInputContexts());
    List<OutputContext> outputContexts = new LinkedList<OutputContext>();
    outputContexts.addAll(lio.getOutputContexts());
    lio.cleanup();
    assertTrue(procContext.getUserPayload() == null);
    assertTrue(procContext.getObjectRegistry() == null);
    for (InputContext inputContext : inputContexts) {
        assertTrue(inputContext.getUserPayload() == null);
        assertTrue(inputContext.getObjectRegistry() == null);
    }
    for (OutputContext outputContext : outputContexts) {
        assertTrue(outputContext.getUserPayload() == null);
        assertTrue(outputContext.getObjectRegistry() == null);
    }
    boolean localMode = lio.tezConf.getBoolean(TezConfiguration.TEZ_LOCAL_MODE, TezConfiguration.TEZ_LOCAL_MODE_DEFAULT);
    if (localMode) {
        assertEquals(1, lio.inputSpecs.size());
        assertEquals(1, lio.outputSpecs.size());
        assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
    } else {
        assertEquals(0, lio.inputSpecs.size());
        assertEquals(0, lio.outputSpecs.size());
        assertTrue(lio.groupInputSpecs == null || lio.groupInputSpecs.size() == 0);
    }
    assertEquals(0, lio.inputsMap.size());
    assertEquals(0, lio.inputContextMap.size());
    assertEquals(0, lio.outputsMap.size());
    assertEquals(0, lio.outputContextMap.size());
    assertNull(lio.groupInputsMap);
    assertNull(lio.processor);
    assertNull(lio.processorContext);
    assertEquals(0, lio.runInputMap.size());
    assertEquals(0, lio.runOutputMap.size());
    assertEquals(0, lio.eventsToBeProcessed.size());
    assertNull(lio.eventRouterThread);
}

Example 5 with ProcessorContext

use of org.apache.tez.runtime.api.ProcessorContext in project tez by apache.

the class TestWeightedScalingMemoryDistributor method testWeightedScalingNonConcurrent.

@Test(timeout = 5000)
public void testWeightedScalingNonConcurrent() throws TezException {
    Configuration conf = new Configuration(this.conf);
    conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_INPUT_OUTPUT_CONCURRENT, false);
    conf.setBoolean(TezConfiguration.TEZ_TASK_SCALE_MEMORY_NON_CONCURRENT_INPUTS_ENABLED, true);
    conf.setDouble(TezConfiguration.TEZ_TASK_SCALE_MEMORY_RESERVE_FRACTION, 0.2);
    conf.setStrings(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS, WeightedScalingMemoryDistributor.generateWeightStrings(0, 0, 1, 2, 3, 1, 1));
    System.err.println(Joiner.on(",").join(conf.getStringCollection(TezConfiguration.TEZ_TASK_SCALE_MEMORY_WEIGHTED_RATIOS)));
    MemoryDistributor dist = new MemoryDistributor(2, 2, conf);
    dist.setJvmMemory(10000l);
    // First request - ScatterGatherShuffleInput
    MemoryUpdateCallbackForTest e1Callback = new MemoryUpdateCallbackForTest();
    InputContext e1InputContext1 = createTestInputContext();
    InputDescriptor e1InDesc1 = createTestInputDescriptor(OrderedGroupedKVInput.class);
    dist.requestMemory(10000, e1Callback, e1InputContext1, e1InDesc1);
    // Second request - BroadcastInput
    MemoryUpdateCallbackForTest e2Callback = new MemoryUpdateCallbackForTest();
    InputContext e2InputContext2 = createTestInputContext();
    InputDescriptor e2InDesc2 = createTestInputDescriptor(UnorderedKVInput.class);
    dist.requestMemory(10000, e2Callback, e2InputContext2, e2InDesc2);
    // Third request - randomOutput (simulates MROutput)
    MemoryUpdateCallbackForTest e3Callback = new MemoryUpdateCallbackForTest();
    OutputContext e3OutputContext1 = createTestOutputContext();
    OutputDescriptor e3OutDesc1 = createTestOutputDescriptor();
    dist.requestMemory(10000, e3Callback, e3OutputContext1, e3OutDesc1);
    // Fourth request - OnFileSortedOutput
    MemoryUpdateCallbackForTest e4Callback = new MemoryUpdateCallbackForTest();
    OutputContext e4OutputContext2 = createTestOutputContext();
    OutputDescriptor e4OutDesc2 = createTestOutputDescriptor(OrderedPartitionedKVOutput.class);
    dist.requestMemory(10000, e4Callback, e4OutputContext2, e4OutDesc2);
    // Fifth request - Processor
    MemoryUpdateCallbackForTest e5Callback = new MemoryUpdateCallbackForTest();
    ProcessorContext e5ProcContext = createTestProcessortContext();
    ProcessorDescriptor e5ProcDesc = createTestProcessorDescriptor();
    dist.requestMemory(10000, e5Callback, e5ProcContext, e5ProcDesc);
    dist.makeInitialAllocations();
    // Total available: 80% of 10K = 8000
    // 5 requests (weight) - 10K (3), 10K(1), 10K(1), 10K(2), 10K(1)
    // Overlap input and output memory
    assertEquals(5250, e1Callback.assigned);
    assertEquals(1750, e2Callback.assigned);
    assertEquals(2333, e3Callback.assigned);
    assertEquals(4666, e4Callback.assigned);
    assertEquals(1000, e5Callback.assigned);
}